+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-39.ec2.internal ++ [[ -n ip-10-0-2-39.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-39.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 15:03:04.998210330 - disable conntrack on geneve port' I0327 15:03:04.998210330 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0327 15:03:05.062484908 - starting ovnkube-node + echo 'I0327 15:03:05.062484908 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-39.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-39.ec2.internal --init-node ip-10-0-2-39.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-39.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0327 15:03:05.117077 4011 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0327 15:03:05.117118 4011 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-c3e8d4b4b8.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0327 15:03:05.118527 4011 kube.go:419] Waiting for certificate I0327 15:03:05.118567 4011 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0327 15:03:05.118972 4011 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0327 15:03:05.119649 4011 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 15:03:05.176635 4011 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:03:05.176651 4011 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:03:05.184521 4011 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:03:05.184607 4011 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-sq22z" I0327 15:03:05.195427 4011 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-sq22z" I0327 15:03:05.195487 4011 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:03:06.119256 4011 kube.go:426] Certificate found I0327 15:03:06.119795 4011 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 15:03:06.121175 4011 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0327 15:03:06.193030 4011 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 15:03:06.200028 4011 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-28 14:58:05 +0000 UTC" deadline="2026-03-28 11:45:49.008416354 +0000 UTC" I0327 15:03:06.200097 4011 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h42m42.808333931s" I0327 15:03:06.211812 4011 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0327 15:03:06.211964 4011 factory.go:531] Starting watch factory I0327 15:03:06.212101 4011 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.212139 4011 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.212234 4011 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.212315 4011 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.212507 4011 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.212598 4011 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.212658 4011 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.212676 4011 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.212946 4011 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.212972 4011 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.213126 4011 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.213184 4011 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.215238 4011 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0327 15:03:06.218774 4011 controller_manager.go:368] Starting the ovnkube controller I0327 15:03:06.218784 4011 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-39.ec2.internal I0327 15:03:06.218824 4011 controller_manager.go:393] NBDB zone sync took: 33.349µs I0327 15:03:06.218832 4011 factory.go:531] Starting watch factory I0327 15:03:06.234168 4011 ovn_northd.go:121] Registering OVN northd metrics I0327 15:03:06.234751 4011 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.237375 4011 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.237487 4011 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0327 15:03:06.237442 4011 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.238925 4011 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.241277 4011 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0327 15:03:06.241370 4011 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0327 15:03:06.243313 4011 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.246559 4011 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:06.256938 4011 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0327 15:03:06.318068 4011 factory.go:1890] *v1.Pod informer cache synced successfully I0327 15:03:06.318081 4011 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 15:03:06.318085 4011 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 15:03:06.318088 4011 factory.go:1890] *v1.Node informer cache synced successfully I0327 15:03:06.318091 4011 factory.go:1890] *v1.Service informer cache synced successfully I0327 15:03:06.318094 4011 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 15:03:06.318187 4011 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:06.318205 4011 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:06.318187 4011 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:06.318258 4011 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:06.324662 4011 factory.go:1890] *v1.Service informer cache synced successfully I0327 15:03:06.324673 4011 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 15:03:06.324677 4011 factory.go:1890] *v1.Pod informer cache synced successfully I0327 15:03:06.324679 4011 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 15:03:06.324682 4011 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 15:03:06.324685 4011 factory.go:1890] *v1.Node informer cache synced successfully I0327 15:03:06.342311 4011 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:06.362587 4011 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:06.423997 4011 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 15:03:06.424007 4011 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 15:03:06.424140 4011 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.424158 4011 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.430650 4011 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 15:03:06.430661 4011 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 15:03:06.443033 4011 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.529957 4011 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 15:03:06.530026 4011 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.530050 4011 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.536650 4011 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 15:03:06.549261 4011 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.636251 4011 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 15:03:06.636320 4011 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.636333 4011 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.642750 4011 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 15:03:06.655564 4011 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.742070 4011 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 15:03:06.742130 4011 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.742143 4011 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.748795 4011 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 15:03:06.768571 4011 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.848473 4011 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 15:03:06.848587 4011 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.848605 4011 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.854791 4011 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 15:03:06.867716 4011 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:06.954569 4011 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 15:03:06.954767 4011 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:03:06.954781 4011 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:03:06.960103 4011 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 15:03:06.973341 4011 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:03:07.061052 4011 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 15:03:07.061182 4011 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:07.061250 4011 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:07.061269 4011 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:07.061225 4011 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:07.066338 4011 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 15:03:07.105160 4011 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:07.146645 4011 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:07.166564 4011 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 15:03:07.166572 4011 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 15:03:07.166580 4011 factory.go:636] Watch Factory start up complete, took: 954.621504ms I0327 15:03:07.166664 4011 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0327 15:03:07.172011 4011 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 15:03:07.172025 4011 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 15:03:07.172032 4011 factory.go:636] Watch Factory start up complete, took: 953.198993ms I0327 15:03:07.172040 4011 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-39.ec2.internal" zone I0327 15:03:07.212921 4011 default_node_network_controller.go:709] Initializing the default node network controller I0327 15:03:07.269082 4011 udn_isolation.go:101] Starting UDN host isolation manager I0327 15:03:07.275489 4011 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0327 15:03:07.341183 4011 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0327 15:03:07.341249 4011 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0327 15:03:07.341264 4011 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0327 15:03:07.371973 4011 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0327 15:03:07.372033 4011 default_node_network_controller.go:808] Node ip-10-0-2-39.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0327 15:03:07.377695 4011 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.39"] k8s.ovn.org/zone-name:ip-10-0-2-39.ec2.internal] on node ip-10-0-2-39.ec2.internal I0327 15:03:07.398303 4011 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0327 15:03:07.403449 4011 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0327 15:03:07.403622 4011 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0327 15:03:07.403783 4011 gateway_init.go:263] Preparing Gateway I0327 15:03:07.403797 4011 gateway_shared_intf.go:1658] Creating new gateway I0327 15:03:07.422136 4011 controller_manager.go:427] Waiting for node in zone sync took: 250.085052ms I0327 15:03:07.427358 4011 controller_manager.go:608] Switching to transit router for layer2 networks I0327 15:03:07.427372 4011 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-39.ec2.internal I0327 15:03:07.443666 4011 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 15:03:07.449146 4011 controller_manager.go:288] SCTP support detected in OVN I0327 15:03:07.449742 4011 services_controller.go:69] Creating services controller for network=default I0327 15:03:07.449813 4011 udn_enabled_svc.go:80] Waiting for service informer to sync I0327 15:03:07.449879 4011 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0327 15:03:07.449904 4011 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0327 15:03:07.449923 4011 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0327 15:03:07.449933 4011 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 15:03:07.450080 4011 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0327 15:03:07.450133 4011 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0327 15:03:07.450157 4011 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 44.489µs I0327 15:03:07.450168 4011 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 24.361µs I0327 15:03:07.450176 4011 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 42.427µs I0327 15:03:07.450182 4011 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0327 15:03:07.450192 4011 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0327 15:03:07.450207 4011 default_network_controller.go:321] Starting the default network controller I0327 15:03:07.450216 4011 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 11.358µs I0327 15:03:07.450236 4011 udn_enabled_svc.go:95] Performing full resync I0327 15:03:07.450494 4011 udn_enabled_svc.go:99] Waiting for handler to sync I0327 15:03:07.450503 4011 udn_enabled_svc.go:104] Starting worker I0327 15:03:07.457934 4011 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0327 15:03:07.457989 4011 repair.go:33] Syncing exgw routes took 38.591µs I0327 15:03:07.458007 4011 default_network_controller.go:407] Starting all the Watchers... I0327 15:03:07.458137 4011 namespace.go:98] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] adding namespace I0327 15:03:07.458153 4011 namespace.go:98] [openshift-backplane-cee] adding namespace I0327 15:03:07.458163 4011 namespace.go:98] [default] adding namespace I0327 15:03:07.458174 4011 namespace.go:98] [openshift-config-operator] adding namespace I0327 15:03:07.458176 4011 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0327 15:03:07.458177 4011 namespace.go:98] [kube-node-lease] adding namespace I0327 15:03:07.458175 4011 namespace.go:98] [openshift-network-console] adding namespace I0327 15:03:07.458195 4011 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0327 15:03:07.458196 4011 namespace.go:98] [openshift-package-operator] adding namespace I0327 15:03:07.458199 4011 namespace.go:98] [openshift-apiserver] adding namespace I0327 15:03:07.458197 4011 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0327 15:03:07.458207 4011 namespace.go:98] [openshift-ingress-canary] adding namespace I0327 15:03:07.458215 4011 namespace.go:98] [openshift-operators] adding namespace I0327 15:03:07.458217 4011 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0327 15:03:07.458137 4011 namespace.go:98] [openshift-backplane-mobb] adding namespace I0327 15:03:07.458838 4011 namespace.go:102] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] adding namespace took 688.973µs I0327 15:03:07.458855 4011 namespace.go:98] [openshift-cluster-version] adding namespace I0327 15:03:07.459336 4011 namespace.go:102] [openshift-cluster-version] adding namespace took 476.3µs I0327 15:03:07.459350 4011 namespace.go:98] [openshift-image-registry] adding namespace I0327 15:03:07.459904 4011 namespace.go:102] [openshift-image-registry] adding namespace took 547.937µs I0327 15:03:07.459919 4011 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0327 15:03:07.460420 4011 namespace.go:102] [openshift-backplane-cee] adding namespace took 2.260935ms I0327 15:03:07.460430 4011 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0327 15:03:07.460816 4011 namespace.go:102] [default] adding namespace took 2.644436ms I0327 15:03:07.460826 4011 namespace.go:98] [kube-public] adding namespace I0327 15:03:07.461344 4011 namespace.go:102] [openshift-config-operator] adding namespace took 3.160679ms I0327 15:03:07.461356 4011 namespace.go:98] [openshift-controller-manager] adding namespace I0327 15:03:07.461809 4011 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 3.62361ms I0327 15:03:07.461818 4011 namespace.go:98] [openshift-route-controller-manager] adding namespace I0327 15:03:07.462276 4011 namespace.go:102] [kube-node-lease] adding namespace took 4.084979ms I0327 15:03:07.462291 4011 namespace.go:98] [openshift-ingress-operator] adding namespace I0327 15:03:07.462716 4011 namespace.go:102] [openshift-network-console] adding namespace took 4.522642ms I0327 15:03:07.462728 4011 namespace.go:98] [openshift-backplane-csm] adding namespace I0327 15:03:07.463228 4011 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 5.005358ms I0327 15:03:07.463242 4011 namespace.go:98] [openshift-machine-api] adding namespace I0327 15:03:07.463698 4011 namespace.go:102] [openshift-package-operator] adding namespace took 5.49133ms I0327 15:03:07.463713 4011 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0327 15:03:07.464173 4011 namespace.go:102] [openshift-apiserver] adding namespace took 5.963569ms I0327 15:03:07.464188 4011 namespace.go:98] [openshift-config-managed] adding namespace I0327 15:03:07.464609 4011 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.393552ms I0327 15:03:07.464622 4011 namespace.go:98] [openshift-service-ca-operator] adding namespace I0327 15:03:07.465095 4011 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.881547ms I0327 15:03:07.465109 4011 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0327 15:03:07.465526 4011 namespace.go:102] [openshift-operators] adding namespace took 7.300208ms I0327 15:03:07.465554 4011 namespace.go:98] [openshift-network-operator] adding namespace I0327 15:03:07.465901 4011 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.671412ms I0327 15:03:07.465913 4011 namespace.go:98] [openshift-customer-monitoring] adding namespace I0327 15:03:07.466356 4011 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.085528ms I0327 15:03:07.466369 4011 namespace.go:98] [openshift-console-operator] adding namespace I0327 15:03:07.466740 4011 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.81521ms I0327 15:03:07.466753 4011 namespace.go:98] [openshift-network-diagnostics] adding namespace I0327 15:03:07.467138 4011 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.702918ms I0327 15:03:07.467150 4011 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0327 15:03:07.467498 4011 namespace.go:102] [kube-public] adding namespace took 6.667699ms I0327 15:03:07.467509 4011 namespace.go:98] [kube-system] adding namespace I0327 15:03:07.467939 4011 namespace.go:102] [openshift-controller-manager] adding namespace took 6.579019ms I0327 15:03:07.467952 4011 namespace.go:98] [openshift-marketplace] adding namespace I0327 15:03:07.468276 4011 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.454438ms I0327 15:03:07.468295 4011 namespace.go:98] [openshift-monitoring] adding namespace I0327 15:03:07.468750 4011 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.453842ms I0327 15:03:07.468763 4011 namespace.go:98] [openshift-etcd] adding namespace I0327 15:03:07.469147 4011 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.41415ms I0327 15:03:07.469158 4011 namespace.go:98] [openshift-authentication] adding namespace I0327 15:03:07.469517 4011 namespace.go:102] [openshift-machine-api] adding namespace took 6.270662ms I0327 15:03:07.469530 4011 namespace.go:98] [openshift-backplane-srep] adding namespace I0327 15:03:07.470001 4011 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.282842ms I0327 15:03:07.470012 4011 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0327 15:03:07.470388 4011 namespace.go:102] [openshift-config-managed] adding namespace took 6.195698ms I0327 15:03:07.470401 4011 namespace.go:98] [openshift-infra] adding namespace I0327 15:03:07.470836 4011 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.209828ms I0327 15:03:07.470847 4011 namespace.go:98] [openshift-console-user-settings] adding namespace I0327 15:03:07.471317 4011 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.202363ms I0327 15:03:07.471330 4011 namespace.go:98] [openshift-console] adding namespace I0327 15:03:07.471840 4011 namespace.go:102] [openshift-network-operator] adding namespace took 6.278903ms I0327 15:03:07.471854 4011 namespace.go:98] [openshift-ingress] adding namespace I0327 15:03:07.472616 4011 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.697782ms I0327 15:03:07.472629 4011 namespace.go:98] [openshift-deployment-validation-operator] adding namespace W0327 15:03:07.473022 4011 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-s5gkw: pod openshift-network-diagnostics/network-check-target-s5gkw: no pod IPs found W0327 15:03:07.473041 4011 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-sg8bh: pod openshift-network-diagnostics/network-check-target-sg8bh: no pod IPs found I0327 15:03:07.473061 4011 namespace.go:102] [openshift-console-operator] adding namespace took 6.687262ms I0327 15:03:07.473071 4011 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0327 15:03:07.473481 4011 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.723369ms W0327 15:03:07.473889 4011 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-v7xhv: pod kube-system/global-pull-secret-syncer-v7xhv: no pod IPs found I0327 15:03:07.473928 4011 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.773351ms I0327 15:03:07.473939 4011 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0327 15:03:07.474319 4011 namespace.go:102] [kube-system] adding namespace took 6.805273ms I0327 15:03:07.474330 4011 namespace.go:98] [openshift-node] adding namespace I0327 15:03:07.474711 4011 namespace.go:102] [openshift-marketplace] adding namespace took 6.754567ms I0327 15:03:07.474722 4011 namespace.go:98] [openshift-dns] adding namespace I0327 15:03:07.475093 4011 namespace.go:102] [openshift-monitoring] adding namespace took 6.791354ms I0327 15:03:07.475104 4011 namespace.go:98] [openshift-config] adding namespace I0327 15:03:07.475461 4011 namespace.go:102] [openshift-etcd] adding namespace took 6.693547ms I0327 15:03:07.475473 4011 namespace.go:98] [openshift-multus] adding namespace I0327 15:03:07.475837 4011 namespace.go:102] [openshift-authentication] adding namespace took 6.674206ms I0327 15:03:07.475847 4011 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0327 15:03:07.476205 4011 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.67116ms I0327 15:03:07.476216 4011 namespace.go:98] [openshift-kube-apiserver] adding namespace I0327 15:03:07.476561 4011 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.545904ms I0327 15:03:07.476570 4011 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0327 15:03:07.477068 4011 namespace.go:102] [openshift-infra] adding namespace took 6.661882ms I0327 15:03:07.477081 4011 namespace.go:98] [openshift-backplane-cse] adding namespace I0327 15:03:07.477460 4011 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.60707ms I0327 15:03:07.477473 4011 namespace.go:98] [openshift-dns-operator] adding namespace I0327 15:03:07.477879 4011 namespace.go:102] [openshift-console] adding namespace took 6.543529ms I0327 15:03:07.477892 4011 namespace.go:98] [openshift-machine-config-operator] adding namespace I0327 15:03:07.478299 4011 namespace.go:102] [openshift-ingress] adding namespace took 6.441196ms I0327 15:03:07.478309 4011 namespace.go:98] [openshift-logging] adding namespace I0327 15:03:07.478716 4011 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.082116ms I0327 15:03:07.478726 4011 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0327 15:03:07.479144 4011 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.067521ms I0327 15:03:07.479156 4011 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0327 15:03:07.479517 4011 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 5.573294ms I0327 15:03:07.479528 4011 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0327 15:03:07.479899 4011 namespace.go:102] [openshift-node] adding namespace took 5.562652ms I0327 15:03:07.479922 4011 namespace.go:98] [openshift-network-node-identity] adding namespace I0327 15:03:07.480378 4011 namespace.go:102] [openshift-dns] adding namespace took 5.650168ms I0327 15:03:07.480390 4011 namespace.go:98] [openshift-backplane-tam] adding namespace W0327 15:03:07.480787 4011 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-h9rtb: pod openshift-multus/network-metrics-daemon-h9rtb: no pod IPs found W0327 15:03:07.480796 4011 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-r9wb8: pod openshift-multus/network-metrics-daemon-r9wb8: no pod IPs found I0327 15:03:07.480837 4011 namespace.go:102] [openshift-config] adding namespace took 5.726807ms I0327 15:03:07.480851 4011 namespace.go:98] [dedicated-admin] adding namespace I0327 15:03:07.481130 4011 iptables.go:108] Creating table: filter chain: FORWARD I0327 15:03:07.481280 4011 namespace.go:102] [openshift-multus] adding namespace took 5.800673ms I0327 15:03:07.481298 4011 namespace.go:98] [openshift-backplane] adding namespace I0327 15:03:07.481711 4011 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 5.859078ms I0327 15:03:07.481724 4011 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0327 15:03:07.482070 4011 namespace.go:102] [openshift-kube-apiserver] adding namespace took 5.84969ms I0327 15:03:07.482083 4011 namespace.go:98] [openshift-insights] adding namespace I0327 15:03:07.482517 4011 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 5.94257ms I0327 15:03:07.482525 4011 namespace.go:98] [openshift-operators-redhat] adding namespace I0327 15:03:07.483047 4011 namespace.go:102] [openshift-backplane-cse] adding namespace took 5.961051ms I0327 15:03:07.483064 4011 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0327 15:03:07.483386 4011 namespace.go:102] [openshift-dns-operator] adding namespace took 5.906104ms I0327 15:03:07.483400 4011 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0327 15:03:07.483920 4011 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.023711ms I0327 15:03:07.483932 4011 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0327 15:03:07.484331 4011 namespace.go:102] [openshift-logging] adding namespace took 6.016972ms I0327 15:03:07.484346 4011 namespace.go:98] [openshift-kube-scheduler] adding namespace I0327 15:03:07.484772 4011 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.039023ms I0327 15:03:07.484790 4011 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0327 15:03:07.485228 4011 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.06709ms I0327 15:03:07.485244 4011 namespace.go:98] [openshift-authentication-operator] adding namespace I0327 15:03:07.485665 4011 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.131688ms I0327 15:03:07.485678 4011 namespace.go:98] [openshift] adding namespace I0327 15:03:07.486059 4011 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.130016ms I0327 15:03:07.486489 4011 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.093265ms I0327 15:03:07.486878 4011 namespace.go:102] [dedicated-admin] adding namespace took 6.02246ms I0327 15:03:07.486894 4011 namespace.go:98] [openshift-must-gather-operator] adding namespace I0327 15:03:07.486996 4011 iptables.go:108] Creating table: filter chain: OUTPUT I0327 15:03:07.487288 4011 namespace.go:102] [openshift-backplane] adding namespace took 5.984454ms I0327 15:03:07.487812 4011 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.082612ms I0327 15:03:07.487826 4011 namespace.go:98] [openshift-host-network] adding namespace I0327 15:03:07.488244 4011 namespace.go:102] [openshift-insights] adding namespace took 6.157092ms I0327 15:03:07.488758 4011 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.228417ms I0327 15:03:07.489253 4011 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.184575ms I0327 15:03:07.489828 4011 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.423662ms I0327 15:03:07.490312 4011 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.374251ms I0327 15:03:07.490782 4011 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.429744ms I0327 15:03:07.491213 4011 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.418896ms I0327 15:03:07.491752 4011 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.502414ms I0327 15:03:07.491765 4011 namespace.go:98] [openshift-apiserver-operator] adding namespace I0327 15:03:07.492248 4011 namespace.go:102] [openshift] adding namespace took 6.564947ms I0327 15:03:07.492805 4011 namespace.go:102] [openshift-must-gather-operator] adding namespace took 5.906551ms I0327 15:03:07.492948 4011 gateway_shared_intf.go:1787] Gateway Creation Complete I0327 15:03:07.492961 4011 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-39.ec2.internal","mac-address":"0e:a8:07:a1:c2:79","ip-addresses":["10.0.2.39/24"],"ip-address":"10.0.2.39/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:a58331b7-d2a4-46da-8dda-15dcf255d3b7 k8s.ovn.org/node-encap-ips:["10.0.2.39"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.39/24"} k8s.ovn.org/zone-name:ip-10-0-2-39.ec2.internal] on node ip-10-0-2-39.ec2.internal I0327 15:03:07.493295 4011 namespace.go:102] [openshift-host-network] adding namespace took 5.465355ms I0327 15:03:07.493714 4011 namespace.go:102] [openshift-apiserver-operator] adding namespace took 1.944689ms I0327 15:03:07.494066 4011 zone_ic_handler.go:186] Time taken to create transit switch: 267.715µs I0327 15:03:07.494142 4011 master.go:532] Adding or Updating local node "ip-10-0-2-39.ec2.internal" for network "default" I0327 15:03:07.494169 4011 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-176.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-176.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-176.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-176.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-176.ec2.internal I0327 15:03:07.497581 4011 default_network_controller.go:806] Node add failed for ip-10-0-2-39.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-39.ec2.internal, error getting gateway config for node ip-10-0-2-39.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-39.ec2.internal", failed to update chassis to local for local node ip-10-0-2-39.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-39.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-39.ec2.internal] E0327 15:03:07.497622 4011 obj_retry.go:541] Failed to create *v1.Node ip-10-0-2-39.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-39.ec2.internal, error getting gateway config for node ip-10-0-2-39.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-39.ec2.internal", failed to update chassis to local for local node ip-10-0-2-39.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-39.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-39.ec2.internal] I0327 15:03:07.497654 4011 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0327 15:03:07.497676 4011 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0327 15:03:07.497690 4011 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0327 15:03:07.497721 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-176.ec2.internal in network "default" I0327 15:03:07.497759 4011 node_tracker.go:235] Node ip-10-0-1-176.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-176.ec2.internal" I0327 15:03:07.497772 4011 node_tracker.go:169] Node ip-10-0-1-176.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:07.497780 4011 services_controller.go:574] Full service sync requested for network=default I0327 15:03:07.497784 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-39.ec2.internal in network "default" I0327 15:03:07.497793 4011 node_tracker.go:235] Node ip-10-0-2-39.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-39.ec2.internal" I0327 15:03:07.497798 4011 node_tracker.go:169] Node ip-10-0-2-39.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:07.497802 4011 services_controller.go:574] Full service sync requested for network=default I0327 15:03:07.497717 4011 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-39.ec2.internal", UID:"02f5515e-37a5-4b8d-90c8-1c821736f48f", APIVersion:"v1", ResourceVersion:"8304", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-39.ec2.internal, error getting gateway config for node ip-10-0-2-39.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-39.ec2.internal", failed to update chassis to local for local node ip-10-0-2-39.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-39.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-39.ec2.internal] I0327 15:03:07.509371 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-39.ec2.internal in network "default" I0327 15:03:07.509423 4011 node_tracker.go:169] Node ip-10-0-2-39.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:07.509436 4011 services_controller.go:574] Full service sync requested for network=default I0327 15:03:07.509451 4011 master.go:532] Adding or Updating local node "ip-10-0-2-39.ec2.internal" for network "default" I0327 15:03:07.512864 4011 default_node_network_controller.go:518] Node connection status = connected W0327 15:03:07.514061 4011 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-r9wb8: pod openshift-multus/network-metrics-daemon-r9wb8: no pod IPs found W0327 15:03:07.514077 4011 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-v7xhv: pod kube-system/global-pull-secret-syncer-v7xhv: no pod IPs found W0327 15:03:07.514086 4011 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-s5gkw: pod openshift-network-diagnostics/network-check-target-s5gkw: no pod IPs found I0327 15:03:07.515032 4011 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-39.ec2.internal for the network default I0327 15:03:07.597872 4011 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0327 15:03:07.597888 4011 services_controller.go:208] Setting up event handlers for services for network=default I0327 15:03:07.597928 4011 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0327 15:03:07.597964 4011 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0327 15:03:07.597974 4011 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0327 15:03:07.698630 4011 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0327 15:03:07.698644 4011 repair.go:60] Starting repairing loop for services I0327 15:03:07.698715 4011 repair.go:131] Deleted 0 stale service LBs I0327 15:03:07.698732 4011 repair.go:137] Deleted 0 stale Chassis Template Vars I0327 15:03:07.698745 4011 repair.go:62] Finished repairing loop for services: 101.812µs I0327 15:03:07.698768 4011 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0327 15:03:07.698778 4011 services_controller.go:254] Starting workers for network=default I0327 15:03:07.699146 4011 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-v7xhv] creating logical port kube-system_global-pull-secret-syncer-v7xhv for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:07.699256 4011 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-r9wb8] creating logical port openshift-multus_network-metrics-daemon-r9wb8 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:07.699309 4011 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-h9rtb, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-h9rtb: suppressed error logged: pod openshift-multus/network-metrics-daemon-h9rtb: no pod IPs found I0327 15:03:07.699314 4011 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-v7xhv I0327 15:03:07.699411 4011 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-r9wb8 I0327 15:03:07.699482 4011 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-sg8bh, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-sg8bh: suppressed error logged: pod openshift-network-diagnostics/network-check-target-sg8bh: no pod IPs found I0327 15:03:07.699529 4011 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-s5gkw] creating logical port openshift-network-diagnostics_network-check-target-s5gkw for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:07.699621 4011 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-s5gkw I0327 15:03:07.723937 4011 pod.go:62] [kube-system/global-pull-secret-syncer-v7xhv] pod update took 24.716391ms I0327 15:03:07.723961 4011 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-v7xhv] addLogicalPort annotation time took 24.741948ms I0327 15:03:07.724854 4011 pods.go:271] [kube-system/global-pull-secret-syncer-v7xhv] addLogicalPort took 25.718663ms, libovsdb time 694.621µs I0327 15:03:07.727235 4011 pod.go:62] [openshift-network-diagnostics/network-check-target-s5gkw] pod update took 27.631826ms I0327 15:03:07.727250 4011 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-s5gkw] addLogicalPort annotation time took 27.648101ms I0327 15:03:07.727236 4011 pod.go:62] [openshift-multus/network-metrics-daemon-r9wb8] pod update took 27.882733ms I0327 15:03:07.727337 4011 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-r9wb8] addLogicalPort annotation time took 27.983201ms I0327 15:03:07.728204 4011 pods.go:271] [openshift-network-diagnostics/network-check-target-s5gkw] addLogicalPort took 28.682946ms, libovsdb time 750.882µs I0327 15:03:07.728294 4011 pods.go:271] [openshift-multus/network-metrics-daemon-r9wb8] addLogicalPort took 29.072593ms, libovsdb time 786.366µs I0327 15:03:07.728798 4011 admin_network_policy_controller.go:227] Starting controller default-network-controller I0327 15:03:07.728818 4011 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0327 15:03:07.728827 4011 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0327 15:03:07.728832 4011 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0327 15:03:07.729010 4011 repair.go:29] Repairing admin network policies took 173.012µs I0327 15:03:07.729106 4011 repair.go:92] Repairing baseline admin network policies took 88.423µs I0327 15:03:07.729564 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0327 15:03:07.729579 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0327 15:03:07.730890 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0327 15:03:07.730909 4011 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0327 15:03:07.730916 4011 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0327 15:03:07.732168 4011 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0327 15:03:07.732187 4011 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0327 15:03:07.732201 4011 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0327 15:03:07.733772 4011 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0327 15:03:07.733792 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0327 15:03:07.733801 4011 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0327 15:03:07.735311 4011 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0327 15:03:07.735329 4011 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0327 15:03:07.735338 4011 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0327 15:03:07.736977 4011 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0327 15:03:07.736996 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0327 15:03:07.737004 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0327 15:03:07.738528 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0327 15:03:07.738574 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0327 15:03:07.738584 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0327 15:03:07.739121 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0327 15:03:07.739139 4011 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0327 15:03:07.739146 4011 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0327 15:03:07.739689 4011 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0327 15:03:07.739707 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0327 15:03:07.739716 4011 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0327 15:03:07.740245 4011 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0327 15:03:07.740264 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0327 15:03:07.740271 4011 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0327 15:03:07.740622 4011 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0327 15:03:07.740641 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0327 15:03:07.740649 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0327 15:03:07.741187 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0327 15:03:07.741206 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0327 15:03:07.741214 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0327 15:03:07.741861 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0327 15:03:07.741879 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0327 15:03:07.741886 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0327 15:03:07.743273 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0327 15:03:07.743290 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0327 15:03:07.743298 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0327 15:03:07.743785 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0327 15:03:07.743801 4011 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0327 15:03:07.743813 4011 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0327 15:03:07.745202 4011 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0327 15:03:07.745219 4011 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0327 15:03:07.745229 4011 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0327 15:03:07.746325 4011 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0327 15:03:07.746342 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0327 15:03:07.746350 4011 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0327 15:03:07.747058 4011 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0327 15:03:07.747076 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0327 15:03:07.747085 4011 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0327 15:03:07.747591 4011 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0327 15:03:07.747609 4011 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0327 15:03:07.747618 4011 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0327 15:03:07.748615 4011 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0327 15:03:07.748634 4011 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0327 15:03:07.748642 4011 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0327 15:03:07.749634 4011 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0327 15:03:07.749652 4011 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0327 15:03:07.749662 4011 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0327 15:03:07.750978 4011 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0327 15:03:07.750998 4011 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0327 15:03:07.751014 4011 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0327 15:03:07.752145 4011 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0327 15:03:07.752162 4011 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0327 15:03:07.752172 4011 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0327 15:03:07.753201 4011 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0327 15:03:07.753219 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0327 15:03:07.753227 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0327 15:03:07.753758 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0327 15:03:07.753790 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0327 15:03:07.754196 4011 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0327 15:03:07.754212 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 15:03:07.754909 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0327 15:03:07.754928 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0327 15:03:07.754935 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0327 15:03:07.755468 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0327 15:03:07.755486 4011 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0327 15:03:07.755493 4011 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0327 15:03:07.756718 4011 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0327 15:03:07.756738 4011 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0327 15:03:07.756748 4011 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0327 15:03:07.757915 4011 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0327 15:03:07.757932 4011 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0327 15:03:07.757956 4011 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 15:03:07.758741 4011 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0327 15:03:07.763096 4011 egressfirewall.go:319] Starting EgressFirewall controller I0327 15:03:07.763119 4011 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0327 15:03:07.763132 4011 controller.go:133] Adding controller egress-firewall-controller event handlers I0327 15:03:07.763135 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0327 15:03:07.763154 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0327 15:03:07.763159 4011 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0327 15:03:07.763163 4011 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0327 15:03:07.763408 4011 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0327 15:03:07.763423 4011 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0327 15:03:07.763431 4011 egressqos.go:193] Setting up event handlers for EgressQoS I0327 15:03:07.763522 4011 egressfirewall.go:628] Syncing node "ip-10-0-1-176.ec2.internal" for egress firewall I0327 15:03:07.763552 4011 egressfirewall.go:628] Syncing node "ip-10-0-2-39.ec2.internal" for egress firewall I0327 15:03:07.763589 4011 egressqos.go:245] Starting EgressQoS Controller I0327 15:03:07.763599 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0327 15:03:07.763605 4011 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0327 15:03:07.763610 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0327 15:03:07.763614 4011 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0327 15:03:07.763619 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0327 15:03:07.763624 4011 shared_informer.go:356] "Caches are synced" controller="egressqos" I0327 15:03:07.763627 4011 egressqos.go:259] Repairing EgressQoSes I0327 15:03:07.763631 4011 egressqos.go:399] Starting repairing loop for egressqos I0327 15:03:07.763696 4011 egressqos.go:401] Finished repairing loop for egressqos: 64.962µs I0327 15:03:07.763710 4011 egressservice_zone.go:132] Setting up event handlers for Egress Services I0327 15:03:07.763741 4011 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-39.ec2.internal I0327 15:03:07.763778 4011 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-39.ec2.internal : 37.91µs I0327 15:03:07.763819 4011 egressservice_zone.go:210] Starting Egress Services Controller I0327 15:03:07.763832 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 15:03:07.763840 4011 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 15:03:07.763846 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 15:03:07.763850 4011 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 15:03:07.763855 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 15:03:07.763860 4011 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 15:03:07.763865 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0327 15:03:07.763870 4011 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0327 15:03:07.763874 4011 egressservice_zone.go:228] Repairing Egress Services I0327 15:03:07.764121 4011 master_controller.go:88] Starting Admin Policy Based Route Controller I0327 15:03:07.764130 4011 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 15:03:07.764194 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-176.ec2.internal I0327 15:03:07.764204 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-176.ec2.internal: 11.118µs I0327 15:03:07.764210 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-39.ec2.internal I0327 15:03:07.764214 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-39.ec2.internal: 3.992µs I0327 15:03:07.764230 4011 default_network_controller.go:568] Completing all the Watchers took 306.216978ms I0327 15:03:07.764243 4011 default_network_controller.go:572] Starting unidling controllers I0327 15:03:07.764255 4011 unidle.go:46] Registering OVN SB ControllerEvent handler I0327 15:03:07.764260 4011 unidle.go:63] Populating Initial ContollerEvent events I0327 15:03:07.764269 4011 unidle.go:79] Setting up event handlers for services I0327 15:03:07.764310 4011 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0327 15:03:08.013863 4011 default_node_network_controller.go:518] Node connection status = connected I0327 15:03:08.513480 4011 default_node_network_controller.go:518] Node connection status = connected I0327 15:03:09.000228 4011 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0327 15:03:09.013735 4011 default_node_network_controller.go:518] Node connection status = connected I0327 15:03:09.027933 4011 bridgeconfig.go:537] Gateway is ready I0327 15:03:09.027948 4011 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0327 15:03:09.037851 4011 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.39. Updating OVN encap IP. I0327 15:03:09.042156 4011 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.39 - it is already configured I0327 15:03:09.042171 4011 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.39/24:{} 172.20.0.1/32:{}]. Updating annotations. I0327 15:03:09.042531 4011 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.39/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-39.ec2.internal","mac-address":"0e:a8:07:a1:c2:79","ip-addresses":["10.0.2.39/24"],"ip-address":"10.0.2.39/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:a58331b7-d2a4-46da-8dda-15dcf255d3b7 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.39/24"}] on node ip-10-0-2-39.ec2.internal I0327 15:03:09.059125 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-39.ec2.internal I0327 15:03:09.059136 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-39.ec2.internal in network "default" I0327 15:03:09.059143 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-39.ec2.internal: 22.234µs I0327 15:03:09.059179 4011 egressfirewall.go:628] Syncing node "ip-10-0-2-39.ec2.internal" for egress firewall I0327 15:03:09.059216 4011 node_tracker.go:169] Node ip-10-0-2-39.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:09.059234 4011 services_controller.go:574] Full service sync requested for network=default I0327 15:03:09.059384 4011 master.go:532] Adding or Updating local node "ip-10-0-2-39.ec2.internal" for network "default" I0327 15:03:09.059931 4011 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.39 I0327 15:03:09.060040 4011 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-39.ec2.internal I0327 15:03:09.064127 4011 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-39.ec2.internal. Updating no re-route policies I0327 15:03:09.064277 4011 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-5hsmj I0327 15:03:09.064287 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-39.ec2.internal I0327 15:03:09.064295 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-tmgl9 I0327 15:03:09.064302 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-39.ec2.internal I0327 15:03:09.064308 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-tmgl9 I0327 15:03:09.064321 4011 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-5hsmj I0327 15:03:09.064335 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-6jg7r I0327 15:03:09.064334 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-9r928 I0327 15:03:09.064343 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-tmgl9 after 0 failed attempt(s) I0327 15:03:09.064339 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-xtw5z I0327 15:03:09.064346 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-6jg7r I0327 15:03:09.064349 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-h9rtb I0327 15:03:09.064353 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-9r928 I0327 15:03:09.064357 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-xtw5z I0327 15:03:09.064360 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-r9wb8 I0327 15:03:09.064365 4011 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-39.ec2.internal I0327 15:03:09.064366 4011 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-gzqwm I0327 15:03:09.064365 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-h9rtb I0327 15:03:09.064372 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-r9wb8 I0327 15:03:09.064371 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-7v95q I0327 15:03:09.064379 4011 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-gzqwm I0327 15:03:09.064379 4011 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-39.ec2.internal I0327 15:03:09.064348 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-s5gkw I0327 15:03:09.064387 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-7v95q I0327 15:03:09.064392 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-s5gkw I0327 15:03:09.064403 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-7v95q after 0 failed attempt(s) I0327 15:03:09.064354 4011 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-5hsmj after 0 failed attempt(s) I0327 15:03:09.064381 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-xtw5z after 0 failed attempt(s) I0327 15:03:09.064415 4011 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-r9wb8] creating logical port openshift-multus_network-metrics-daemon-r9wb8 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:09.064419 4011 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-h9rtb, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-h9rtb: suppressed error logged: pod openshift-multus/network-metrics-daemon-h9rtb: no pod IPs found I0327 15:03:09.064434 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-xbjzg I0327 15:03:09.064448 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-xbjzg I0327 15:03:09.064453 4011 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-gzqwm after 0 failed attempt(s) I0327 15:03:09.064348 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-5srjv I0327 15:03:09.064472 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-sg8bh I0327 15:03:09.064480 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-5srjv I0327 15:03:09.064482 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-sg8bh I0327 15:03:09.064498 4011 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-s5gkw] creating logical port openshift-network-diagnostics_network-check-target-s5gkw for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:09.064519 4011 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-sg8bh, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-sg8bh: suppressed error logged: pod openshift-network-diagnostics/network-check-target-sg8bh: no pod IPs found I0327 15:03:09.064481 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-xbjzg after 0 failed attempt(s) I0327 15:03:09.064324 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-39.ec2.internal after 0 failed attempt(s) I0327 15:03:09.064369 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-6jg7r after 0 failed attempt(s) I0327 15:03:09.064549 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-cp8np I0327 15:03:09.064405 4011 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-39.ec2.internal after 0 failed attempt(s) I0327 15:03:09.064357 4011 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-v7xhv I0327 15:03:09.064565 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-cp8np I0327 15:03:09.064570 4011 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-v7xhv I0327 15:03:09.064582 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-cp8np after 0 failed attempt(s) I0327 15:03:09.064606 4011 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-v7xhv] creating logical port kube-system_global-pull-secret-syncer-v7xhv for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:09.064502 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-5srjv after 0 failed attempt(s) I0327 15:03:09.064380 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-9r928 after 0 failed attempt(s) I0327 15:03:09.065288 4011 pods.go:271] [openshift-multus/network-metrics-daemon-r9wb8] addLogicalPort took 885.399µs, libovsdb time 280.792µs I0327 15:03:09.065304 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-r9wb8 after 0 failed attempt(s) I0327 15:03:09.065615 4011 pods.go:271] [kube-system/global-pull-secret-syncer-v7xhv] addLogicalPort took 1.027537ms, libovsdb time 516.313µs I0327 15:03:09.065627 4011 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-v7xhv after 0 failed attempt(s) I0327 15:03:09.065684 4011 pods.go:271] [openshift-network-diagnostics/network-check-target-s5gkw] addLogicalPort took 1.19538ms, libovsdb time 547.693µs I0327 15:03:09.065695 4011 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-s5gkw after 0 failed attempt(s) I0327 15:03:09.076723 4011 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0327 15:03:09.086600 4011 iptables.go:108] Creating table: mangle chain: OUTPUT I0327 15:03:09.089671 4011 iptables.go:108] Creating table: nat chain: OUTPUT I0327 15:03:09.115839 4011 iptables.go:108] Creating table: nat chain: PREROUTING I0327 15:03:09.210967 4011 iptables.go:108] Creating table: filter chain: FORWARD I0327 15:03:09.226620 4011 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0327 15:03:09.226733 4011 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0327 15:03:09.226779 4011 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0327 15:03:09.226890 4011 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0327 15:03:09.226946 4011 gateway_init.go:325] Gateway and management port readiness took 1.716900031s I0327 15:03:09.226970 4011 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0327 15:03:09.227002 4011 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0327 15:03:09.227023 4011 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0327 15:03:09.227032 4011 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 15:03:09.227099 4011 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0327 15:03:09.227220 4011 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 68.34µs I0327 15:03:09.227443 4011 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0327 15:03:09.227466 4011 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 13.912µs I0327 15:03:09.227474 4011 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 31.217µs I0327 15:03:09.227480 4011 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0327 15:03:09.227488 4011 nad_controller.go:173] [node-nad-controller NAD controller]: started I0327 15:03:09.227493 4011 default_node_network_controller.go:898] Starting the default node network controller I0327 15:03:09.227498 4011 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0327 15:03:09.227573 4011 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 12.776µs I0327 15:03:09.227678 4011 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0327 15:03:09.227687 4011 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0327 15:03:09.227708 4011 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0004a9e00 0xc0002f0c40 0xc0002f0cb0 0xc0002f0d20 0xc0002f0d90 0xc0002f0e70 0xc0002f0ee0 0xc00015b180 0xc0002f0f50 0xc0002f0fc0 0xc00043a230 0xc006137e30 0xc0002f1030 map[0x2f67040:0xc00218e600 0x2f67580:0xc00218e780 0x2f68000:0xc00278dec0 0x2f682a0:0xc00218e5a0 0x2f687e0:0xc00218e660 0x2f68a80:0xc00218e6c0 0x2f68d20:0xc00218e720 0x2f69500:0xc00218e7e0 0x2f697a0:0xc00218e840 0x2f93d40:0xc001a5e300 0x2f940e0:0xc00238c300 0x2f94bc0:0xc000c1aba0 0x2f96520:0xc001a5e000 0x2f97000:0xc00238c360 0x2f97e80:0xc001a5e060] 0xc0004fa770 0} I0327 15:03:09.227760 4011 gateway.go:151] Starting gateway service sync I0327 15:03:09.228201 4011 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0327 15:03:09.230194 4011 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0327 15:03:09.256033 4011 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0327 15:03:09.257066 4011 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0327 15:03:09.258165 4011 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0327 15:03:09.291117 4011 gateway.go:168] Gateway service sync done. Time taken: 63.347607ms I0327 15:03:09.291661 4011 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32644 I0327 15:03:09.291825 4011 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0004a9e00 0xc0002f0c40 0xc0002f0cb0 0xc0002f0d20 0xc0002f0d90 0xc0002f0e70 0xc0002f0ee0 0xc00015b180 0xc0002f0f50 0xc0002f0fc0 0xc00043a230 0xc006137e30 0xc0002f1030 map[0x2f67040:0xc00218e600 0x2f67580:0xc00218e780 0x2f68000:0xc00278dec0 0x2f682a0:0xc00218e5a0 0x2f687e0:0xc00218e660 0x2f68a80:0xc00218e6c0 0x2f68d20:0xc00218e720 0x2f69500:0xc00218e7e0 0x2f697a0:0xc00218e840 0x2f93d40:0xc001a5e300 0x2f940e0:0xc00238c300 0x2f94bc0:0xc000c1aba0 0x2f96520:0xc001a5e000 0x2f97000:0xc00238c360 0x2f97e80:0xc001a5e060] 0xc0004fa770 0} I0327 15:03:09.292279 4011 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc0004a9e00 0xc0002f0c40 0xc0002f0cb0 0xc0002f0d20 0xc0002f0d90 0xc0002f0e70 0xc0002f0ee0 0xc00015b180 0xc0002f0f50 0xc0002f0fc0 0xc00043a230 0xc006137e30 0xc0002f1030 map[0x2f67040:0xc00218e600 0x2f67580:0xc00218e780 0x2f68000:0xc00278dec0 0x2f682a0:0xc00218e5a0 0x2f687e0:0xc00218e660 0x2f68a80:0xc00218e6c0 0x2f68d20:0xc00218e720 0x2f69500:0xc00218e7e0 0x2f697a0:0xc00218e840 0x2f93d40:0xc001a5e300 0x2f940e0:0xc00238c300 0x2f94bc0:0xc000c1aba0 0x2f96520:0xc001a5e000 0x2f97000:0xc00238c360 0x2f97e80:0xc001a5e060] 0xc0004fa770 0} I0327 15:03:09.292310 4011 gateway.go:519] Reconciling gateway with updates I0327 15:03:09.297947 4011 gateway.go:326] Spawning Conntrack Rule Check Thread I0327 15:03:09.297967 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:03:09.297974 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:03:09.297984 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:03:09.297989 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:03:09.297996 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 15:03:09.297997 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:03:09.298008 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:09.298012 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:03:09.298015 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:09.298024 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:03:09.298023 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:03:09.298039 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:03:09.298044 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 15:03:09.298049 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:03:09.298035 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:03:09.298064 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:03:09.298062 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 15:03:09.298069 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:03:09.298076 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:03:09.298082 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:03:09.298095 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:03:09.298098 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:03:09.298055 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 15:03:09.298108 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:03:09.298106 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:03:09.298009 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:03:09.298127 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:03:09.298128 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:03:09.298131 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:03:09.298142 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:03:09.298143 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:03:09.298147 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:03:09.298156 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:03:09.298156 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:03:09.298158 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:03:09.298164 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 15:03:09.298162 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 15:03:09.298173 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:03:09.298177 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 15:03:09.298181 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 15:03:09.298180 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 15:03:09.298179 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:03:09.298165 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:03:09.298199 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 15:03:09.298199 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:03:09.298203 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:03:09.298210 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:03:09.298206 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:03:09.298214 4011 default_node_network_controller.go:943] Gateway and management port readiness took 70.519838ms I0327 15:03:09.298219 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 15:03:09.298223 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:03:09.298226 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:03:09.298232 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 15:03:09.298235 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:03:09.298236 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:03:09.298240 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:03:09.298246 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 15:03:09.298249 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:03:09.298251 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:03:09.298251 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:03:09.298256 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 15:03:09.298195 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:03:09.298241 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:03:09.298261 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 15:03:09.298265 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 15:03:09.298266 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 15:03:09.298271 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 15:03:09.298275 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:03:09.298285 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 15:03:09.298285 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:03:09.298222 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 15:03:09.298311 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:03:09.298311 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 15:03:09.298321 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:03:09.298262 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:03:09.298331 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 15:03:09.298335 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 15:03:09.298347 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:03:09.298353 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 15:03:09.298358 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:03:09.298363 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 15:03:09.298334 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 15:03:09.298194 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:03:09.298370 4011 port_claim.go:71] Svc openshift-ingress/router-default: port 30374 is already open, no action needed I0327 15:03:09.298409 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 15:03:09.298411 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:03:09.298416 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 15:03:09.298417 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:03:09.298427 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:03:09.298430 4011 port_claim.go:71] Svc openshift-ingress/router-default: port 31767 is already open, no action needed I0327 15:03:09.298468 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 15:03:09.298422 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:03:09.298484 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 15:03:09.298468 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 15:03:09.298497 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:03:09.298217 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 15:03:09.298216 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 15:03:09.298532 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 15:03:09.298084 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 15:03:09.298640 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:03:09.298656 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:03:09.298224 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 15:03:09.298227 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:03:09.298191 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:03:09.298730 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:03:09.298249 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:03:09.298785 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:03:09.298252 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 15:03:09.298851 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 15:03:09.298857 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 15:03:09.298256 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 15:03:09.298264 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:03:09.298201 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:03:09.298918 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 15:03:09.298185 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:03:09.298930 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:03:09.298211 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 15:03:09.298501 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:03:09.298514 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:03:09.298191 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 15:03:09.299692 4011 node_ip_handler_linux.go:160] Node IP manager is running W0327 15:03:09.334384 4011 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 15:03:09.334529 4011 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 15:03:09.334703 4011 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0327 15:03:09.343284 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 15:03:09.362198 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 15:03:09.362272 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 15:03:09.362291 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 15:03:09.362299 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 15:03:09.398579 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 15:03:09.398589 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 15:03:09.398609 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 15:03:09.398617 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 15:03:09.398625 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 15:03:09.398633 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 15:03:09.398639 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 15:03:09.398645 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 15:03:09.398651 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 15:03:09.398656 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 15:03:10.174071 4011 default_network_controller.go:1008] Node "ip-10-0-1-176.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:11.042643 4011 default_node_network_controller.go:1364] Starting node controller node sync W0327 15:03:11.042651 4011 egressip_healthcheck.go:76] Health checking using insecure connection I0327 15:03:11.042709 4011 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0327 15:03:11.054350 4011 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.687577ms I0327 15:03:11.054599 4011 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0327 15:03:11.054822 4011 egressservice_node.go:93] Setting up event handlers for Egress Services I0327 15:03:11.054990 4011 egressservice_node.go:183] Starting Egress Services Controller I0327 15:03:11.055036 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 15:03:11.055048 4011 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 15:03:11.055054 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 15:03:11.055059 4011 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 15:03:11.055065 4011 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 15:03:11.055069 4011 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 15:03:11.055075 4011 egressservice_node.go:197] Repairing Egress Services I0327 15:03:11.065314 4011 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0327 15:03:11.065341 4011 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 15:03:11.065577 4011 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0327 15:03:11.065590 4011 link_network_manager.go:117] Link manager is running I0327 15:03:11.065602 4011 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0327 15:03:11.065754 4011 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0327 15:03:11.066472 4011 vrf_manager.go:128] VRF manager is running I0327 15:03:11.066828 4011 node_controller_manager.go:437] Removing flows to drop GARP I0327 15:03:11.066841 4011 gateway.go:519] Reconciling gateway with updates I0327 15:03:11.095490 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:03:11.095507 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:03:11.095517 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:03:11.095517 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:03:11.095526 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:03:11.095527 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:03:11.095555 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:03:11.095559 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:03:11.095581 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:03:11.095599 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:03:11.095601 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:03:11.095603 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:03:11.095608 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:03:11.095608 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 15:03:11.095615 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:03:11.095617 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:03:11.095617 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:03:11.095620 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:03:11.095626 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 15:03:11.095629 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:03:11.095633 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:03:11.095638 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 15:03:11.095636 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 15:03:11.095637 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 15:03:11.095653 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 15:03:11.095656 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 15:03:11.095604 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:03:11.095654 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:03:11.095670 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 15:03:11.095674 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 15:03:11.095673 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:03:11.095672 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 15:03:11.095685 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:03:11.095688 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:03:11.095694 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 15:03:11.095694 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 15:03:11.095700 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 15:03:11.095695 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:03:11.095624 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 15:03:11.095705 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 15:03:11.095709 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:03:11.095715 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:03:11.095715 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:03:11.095724 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:03:11.095726 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:03:11.095742 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:03:11.095753 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:03:11.095765 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:03:11.095766 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 15:03:11.095639 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:03:11.095774 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 15:03:11.095789 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:03:11.095800 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 15:03:11.095685 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 15:03:11.095813 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:03:11.095818 4011 port_claim.go:71] Svc openshift-ingress/router-default: port 30374 is already open, no action needed I0327 15:03:11.095818 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:03:11.095825 4011 port_claim.go:71] Svc openshift-ingress/router-default: port 31767 is already open, no action needed I0327 15:03:11.095825 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:03:11.095843 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 15:03:11.095848 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:03:11.095855 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 15:03:11.095860 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:03:11.095862 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:03:11.095867 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:11.095872 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:03:11.095872 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:03:11.095882 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:03:11.095884 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:11.095888 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:03:11.095725 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:03:11.095898 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:03:11.095913 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:03:11.095831 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:03:11.095891 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 15:03:11.095932 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:03:11.095787 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:03:11.095942 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:03:11.095820 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 15:03:11.095959 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 15:03:11.095653 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:03:11.095965 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 15:03:11.095590 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:03:11.095975 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:03:11.095608 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:03:11.096061 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:03:11.096074 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 15:03:11.095705 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:03:11.096089 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:03:11.096091 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:03:11.096096 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 15:03:11.095638 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:03:11.096102 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:03:11.096104 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:03:11.095713 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 15:03:11.095651 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 15:03:11.096193 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 15:03:11.095712 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 15:03:11.095600 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:03:11.096216 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:03:11.095730 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:03:11.095730 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 15:03:11.095806 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:03:11.095803 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 15:03:11.095833 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 15:03:11.095818 4011 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:03:11.096298 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 15:03:11.096303 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:03:11.095871 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:03:11.096338 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 15:03:11.095702 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:03:11.096038 4011 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:03:11.096365 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 15:03:11.132983 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 15:03:11.151384 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 15:03:11.151435 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 15:03:11.151452 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 15:03:11.151460 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 15:03:11.151465 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 15:03:11.151472 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 15:03:11.151477 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 15:03:11.151484 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 15:03:11.151490 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 15:03:11.151495 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 15:03:11.151506 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 15:03:11.151522 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 15:03:11.151528 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 15:03:11.151533 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 15:03:11.151556 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 15:03:11.151566 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 15:03:11.151572 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 15:03:11.190797 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 15:03:11.190817 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 15:03:11.190826 4011 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 15:03:13.190989 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-39.ec2.internal I0327 15:03:13.191032 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-39.ec2.internal: 86.082µs I0327 15:03:13.216649 4011 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5877c497c6-7vmg7] creating logical port openshift-image-registry_image-registry-5877c497c6-7vmg7 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:13.216861 4011 kube.go:256] Updating pod openshift-image-registry/image-registry-5877c497c6-7vmg7 I0327 15:03:13.239360 4011 pod.go:62] [openshift-image-registry/image-registry-5877c497c6-7vmg7] pod update took 22.555083ms I0327 15:03:13.239388 4011 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-5877c497c6-7vmg7] addLogicalPort annotation time took 22.5887ms I0327 15:03:13.240572 4011 pods.go:271] [openshift-image-registry/image-registry-5877c497c6-7vmg7] addLogicalPort took 23.933912ms, libovsdb time 856.781µs I0327 15:03:13.256411 4011 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-fhvsm] creating logical port openshift-dns_dns-default-fhvsm for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:13.256557 4011 kube.go:256] Updating pod openshift-dns/dns-default-fhvsm I0327 15:03:13.259738 4011 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-gn9wz] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-gn9wz for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:13.259850 4011 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-gn9wz I0327 15:03:13.259864 4011 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577060-79dxd for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:13.259940 4011 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd I0327 15:03:13.263650 4011 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-2sgdj] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-2sgdj for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:13.263723 4011 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-2sgdj I0327 15:03:13.263855 4011 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-kqlq7] creating logical port openshift-ingress-canary_ingress-canary-kqlq7 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:13.263962 4011 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-kqlq7 I0327 15:03:13.278353 4011 pod.go:62] [openshift-dns/dns-default-fhvsm] pod update took 21.84177ms I0327 15:03:13.278376 4011 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-fhvsm] addLogicalPort annotation time took 21.866839ms I0327 15:03:13.279622 4011 pods.go:271] [openshift-dns/dns-default-fhvsm] addLogicalPort took 23.225285ms, libovsdb time 958.105µs I0327 15:03:13.281022 4011 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd] pod update took 21.09538ms I0327 15:03:13.281045 4011 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd] addLogicalPort annotation time took 21.114403ms I0327 15:03:13.281390 4011 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-gn9wz] pod update took 21.555672ms I0327 15:03:13.281410 4011 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-gn9wz] addLogicalPort annotation time took 21.580184ms I0327 15:03:13.282137 4011 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd] addLogicalPort took 22.279524ms, libovsdb time 819.745µs I0327 15:03:13.282454 4011 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-gn9wz] addLogicalPort took 22.72598ms, libovsdb time 725.641µs I0327 15:03:13.284803 4011 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-2sgdj] pod update took 21.092133ms I0327 15:03:13.284821 4011 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-2sgdj] addLogicalPort annotation time took 21.111106ms I0327 15:03:13.284893 4011 pod.go:62] [openshift-ingress-canary/ingress-canary-kqlq7] pod update took 20.946664ms I0327 15:03:13.284914 4011 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-kqlq7] addLogicalPort annotation time took 20.968784ms I0327 15:03:13.286119 4011 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-2sgdj] addLogicalPort took 22.476688ms, libovsdb time 1.045971ms I0327 15:03:13.286178 4011 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-xnhr4] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-xnhr4 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:03:13.286240 4011 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-xnhr4 I0327 15:03:13.286391 4011 pods.go:271] [openshift-ingress-canary/ingress-canary-kqlq7] addLogicalPort took 22.542335ms, libovsdb time 1.139205ms I0327 15:03:13.301500 4011 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-xnhr4] pod update took 15.273991ms I0327 15:03:13.301522 4011 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-xnhr4] addLogicalPort annotation time took 15.295794ms I0327 15:03:13.302473 4011 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-xnhr4] addLogicalPort took 16.303679ms, libovsdb time 651.369µs I0327 15:03:13.784610 4011 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd 043bc572f638652965f437dd45f347c5714fc5cdd8409048e1176e001d19834c network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd 043bc572f638652965f437dd45f347c5714fc5cdd8409048e1176e001d19834c network default NAD default] I0327 15:03:13.784639 4011 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-gn9wz 15935c933ec6b5e62e6450f4367ebeeea160c49d74bbacae0b68a8303acb8b6a network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-gn9wz 15935c933ec6b5e62e6450f4367ebeeea160c49d74bbacae0b68a8303acb8b6a network default NAD default] I0327 15:03:13.784614 4011 cni.go:403] [openshift-ingress-canary/ingress-canary-kqlq7 4672bbe36f68ab3ec98fa83ec57b4d2dc9b01c4b7fee1405395e20cbcb4ec6a8 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-kqlq7 4672bbe36f68ab3ec98fa83ec57b4d2dc9b01c4b7fee1405395e20cbcb4ec6a8 network default NAD default] I0327 15:03:13.827191 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577060-79dxd, hostIfaceName: 043bc572f638652, network: default, NAD default, SandboxID: "043bc572f638652965f437dd45f347c5714fc5cdd8409048e1176e001d19834c", PCI device ID: , UID: "d95cee79-79db-4c55-ba86-e0a715ce5157", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0327 15:03:13.833947 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-kqlq7, hostIfaceName: 4672bbe36f68ab3, network: default, NAD default, SandboxID: "4672bbe36f68ab3ec98fa83ec57b4d2dc9b01c4b7fee1405395e20cbcb4ec6a8", PCI device ID: , UID: "986f1fdb-ae9a-40cd-bd13-0a41f11dc364", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0327 15:03:13.843452 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-gn9wz, hostIfaceName: 15935c933ec6b5e, network: default, NAD default, SandboxID: "15935c933ec6b5e62e6450f4367ebeeea160c49d74bbacae0b68a8303acb8b6a", PCI device ID: , UID: "342e922c-92f2-46cb-9dc5-60a6e06ea060", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0327 15:03:13.882519 4011 cni.go:424] [openshift-ingress-canary/ingress-canary-kqlq7 4672bbe36f68ab3ec98fa83ec57b4d2dc9b01c4b7fee1405395e20cbcb4ec6a8 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-kqlq7 4672bbe36f68ab3ec98fa83ec57b4d2dc9b01c4b7fee1405395e20cbcb4ec6a8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:17:ed:58:40:cc\",\"name\":\"4672bbe36f68ab3\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8eaa0b89-9655-49c3-a6f9-f28731a38135\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:03:13.882556 4011 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd 043bc572f638652965f437dd45f347c5714fc5cdd8409048e1176e001d19834c network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd 043bc572f638652965f437dd45f347c5714fc5cdd8409048e1176e001d19834c network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:1f:8c:9c:f2:ce\",\"name\":\"043bc572f638652\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/08d70e8c-5ace-443f-94e8-7561a260d20b\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:03:13.882626 4011 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-gn9wz 15935c933ec6b5e62e6450f4367ebeeea160c49d74bbacae0b68a8303acb8b6a network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-gn9wz 15935c933ec6b5e62e6450f4367ebeeea160c49d74bbacae0b68a8303acb8b6a network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:2b:4d:03:4d:43\",\"name\":\"15935c933ec6b5e\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d980d7e7-123b-46fe-82a5-e13a749d9997\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:03:18.075508 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:03:18.075531 4011 egressfirewall.go:628] Syncing node "ip-10-0-0-80.ec2.internal" for egress firewall I0327 15:03:18.075534 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 32.64µs I0327 15:03:18.075518 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-80.ec2.internal in network "default" I0327 15:03:18.075577 4011 node_tracker.go:220] Node ip-10-0-0-80.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation E0327 15:03:18.075596 4011 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-80.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0327 15:03:18.075617 4011 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-80.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-80.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-80.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-80.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-80.ec2.internal, error parsing annotation for node ip-10-0-0-80.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0327 15:03:18.094619 4011 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-80.ec2.internal annotation: unexpected end of JSON input I0327 15:03:18.094758 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0327 15:03:18.094793 4011 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-80.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0327 15:03:18.100722 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-176.ec2.internal I0327 15:03:18.100741 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-176.ec2.internal: 23.418µs I0327 15:03:18.100742 4011 default_network_controller.go:1008] Node "ip-10-0-1-176.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:18.100759 4011 egressfirewall.go:628] Syncing node "ip-10-0-1-176.ec2.internal" for egress firewall I0327 15:03:18.115736 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:03:18.115739 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:18.115747 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 14.068µs E0327 15:03:18.115776 4011 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-80.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0327 15:03:18.115778 4011 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-80.ec2.internal annotation: unexpected end of JSON input I0327 15:03:18.115803 4011 egressfirewall.go:628] Syncing node "ip-10-0-0-80.ec2.internal" for egress firewall E0327 15:03:18.192240 4011 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-80.ec2.internal annotation: unexpected end of JSON input I0327 15:03:18.192268 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0327 15:03:18.192272 4011 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-80.ec2.internal annotation: unexpected end of JSON input E0327 15:03:18.192325 4011 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-80.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-80.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-80.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-80.ec2.internal" I0327 15:03:18.202400 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0327 15:03:18.274750 4011 egressfirewall.go:628] Syncing node "ip-10-0-0-80.ec2.internal" for egress firewall I0327 15:03:18.274805 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:03:18.274805 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:18.274816 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 10.787µs I0327 15:03:18.290339 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:18.481659 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:18.508847 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:19.980701 4011 default_network_controller.go:1008] Node "ip-10-0-1-176.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:19.980745 4011 egressfirewall.go:628] Syncing node "ip-10-0-1-176.ec2.internal" for egress firewall I0327 15:03:19.980708 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-176.ec2.internal I0327 15:03:19.980771 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-176.ec2.internal: 67.331µs I0327 15:03:21.286508 4011 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-gzqwm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:03:21.286558 4011 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-gzqwm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 15:03:21.533800 4011 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:21.572832 4011 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-gzqwm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:03:21.572870 4011 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-gzqwm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:03:21.578149 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod kube-system/ecr-credential-refresh-initial-gzqwm I0327 15:03:21.578170 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod kube-system/ecr-credential-refresh-initial-gzqwm I0327 15:03:23.523976 4011 default_network_controller.go:1008] Node "ip-10-0-1-176.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:26.802939 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-176.ec2.internal in network "default" I0327 15:03:26.802997 4011 node_tracker.go:235] Node ip-10-0-1-176.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-176.ec2.internal" I0327 15:03:26.803009 4011 default_network_controller.go:1008] Node "ip-10-0-1-176.ec2.internal" in remote zone "ip-10-0-1-176.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:26.803024 4011 node_tracker.go:169] Node ip-10-0-1-176.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:26.803037 4011 services_controller.go:574] Full service sync requested for network=default W0327 15:03:26.803576 4011 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:26.870245 4011 default_network_controller.go:1008] Node "ip-10-0-1-176.ec2.internal" in remote zone "ip-10-0-1-176.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:26.924326 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-176.ec2.internal in network "default" I0327 15:03:26.924375 4011 node_tracker.go:169] Node ip-10-0-1-176.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:26.924388 4011 services_controller.go:574] Full service sync requested for network=default I0327 15:03:26.924421 4011 default_network_controller.go:1008] Node "ip-10-0-1-176.ec2.internal" in remote zone "ip-10-0-1-176.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0327 15:03:26.924686 4011 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:26.925102 4011 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-176.ec2.internal for the network default I0327 15:03:26.926247 4011 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-176.ec2.internal" on network "default" took: 1.15891ms I0327 15:03:27.475098 4011 egressfirewall.go:628] Syncing node "ip-10-0-1-176.ec2.internal" for egress firewall I0327 15:03:27.475124 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-176.ec2.internal I0327 15:03:27.475140 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-176.ec2.internal: 19.752µs I0327 15:03:27.475162 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-176.ec2.internal in network "default" I0327 15:03:27.475194 4011 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-176.ec2.internal", IP: 10.0.1.176 to PMTUD blocking rules I0327 15:03:27.475207 4011 node_tracker.go:169] Node ip-10-0-1-176.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:27.475212 4011 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-176.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:03:27.475219 4011 services_controller.go:574] Full service sync requested for network=default W0327 15:03:27.475598 4011 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:27.481364 4011 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-176.ec2.internal. Updating no re-route policies I0327 15:03:27.493124 4011 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-176.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:03:27.493137 4011 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-176.ec2.internal", IP: 10.0.1.176 to PMTUD blocking rules I0327 15:03:28.463984 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:30.816222 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-176.ec2.internal I0327 15:03:30.816260 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-176.ec2.internal: 43.573µs W0327 15:03:34.565707 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:34.995117 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:03:34.995141 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 27.951µs I0327 15:03:34.995120 4011 egressfirewall.go:628] Syncing node "ip-10-0-0-80.ec2.internal" for egress firewall I0327 15:03:34.995123 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:37.498669 4011 obj_retry.go:300] Retry object setup: *v1.Node ip-10-0-0-80.ec2.internal I0327 15:03:37.498694 4011 obj_retry.go:364] Adding new object: *v1.Node ip-10-0-0-80.ec2.internal I0327 15:03:37.498793 4011 obj_retry.go:375] Retry add failed for *v1.Node ip-10-0-0-80.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-80.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-80.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-80.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-80.ec2.internal I0327 15:03:39.065894 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-9q9gv I0327 15:03:39.065925 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-9q9gv I0327 15:03:39.065966 4011 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-9q9gv, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-9q9gv: suppressed error logged: pod openshift-network-diagnostics/network-check-target-9q9gv: no pod IPs found I0327 15:03:39.065897 4011 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-ktn4j I0327 15:03:39.065979 4011 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-ktn4j I0327 15:03:39.065996 4011 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-ktn4j, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-ktn4j: suppressed error logged: pod openshift-multus/network-metrics-daemon-ktn4j: no pod IPs found W0327 15:03:40.143840 4011 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:03:42.507518 4011 cni.go:403] [kube-system/global-pull-secret-syncer-v7xhv a54d966924b6b55dbbeea3b3dcf8b8cb07048ed691b52bf673a0ffaf337b57b9 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-v7xhv a54d966924b6b55dbbeea3b3dcf8b8cb07048ed691b52bf673a0ffaf337b57b9 network default NAD default] I0327 15:03:42.517671 4011 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-v7xhv, hostIfaceName: a54d966924b6b55, network: default, NAD default, SandboxID: "a54d966924b6b55dbbeea3b3dcf8b8cb07048ed691b52bf673a0ffaf337b57b9", PCI device ID: , UID: "6c0699ba-d2e8-4bce-abf4-d2bc67e01941", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0327 15:03:42.518597 4011 cni.go:403] [openshift-network-diagnostics/network-check-target-s5gkw 9b0a4f0e00236f74aa54545b25d41e7e36fff3d760e9cf2419dcfe0f1f72fe82 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-s5gkw 9b0a4f0e00236f74aa54545b25d41e7e36fff3d760e9cf2419dcfe0f1f72fe82 network default NAD default] I0327 15:03:42.527085 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-s5gkw, hostIfaceName: 9b0a4f0e00236f7, network: default, NAD default, SandboxID: "9b0a4f0e00236f74aa54545b25d41e7e36fff3d760e9cf2419dcfe0f1f72fe82", PCI device ID: , UID: "ea4574c0-be14-451d-973d-b2aa90ee2e3f", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0327 15:03:42.560340 4011 cni.go:424] [kube-system/global-pull-secret-syncer-v7xhv a54d966924b6b55dbbeea3b3dcf8b8cb07048ed691b52bf673a0ffaf337b57b9 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-v7xhv a54d966924b6b55dbbeea3b3dcf8b8cb07048ed691b52bf673a0ffaf337b57b9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:6e:31:0f:3c:9d\",\"name\":\"a54d966924b6b55\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/52a3e5e2-8eee-42cb-8397-48f9d44aa054\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:03:42.577198 4011 cni.go:424] [openshift-network-diagnostics/network-check-target-s5gkw 9b0a4f0e00236f74aa54545b25d41e7e36fff3d760e9cf2419dcfe0f1f72fe82 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-s5gkw 9b0a4f0e00236f74aa54545b25d41e7e36fff3d760e9cf2419dcfe0f1f72fe82 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:fa:aa:4d:79:0d\",\"name\":\"9b0a4f0e00236f7\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ee8a5b44-a387-4e2b-8032-4262612e349c\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:03:46.131457 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-80.ec2.internal in network "default" I0327 15:03:46.131494 4011 node_tracker.go:235] Node ip-10-0-0-80.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-80.ec2.internal" I0327 15:03:46.131515 4011 node_tracker.go:169] Node ip-10-0-0-80.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:46.131513 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "ip-10-0-0-80.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:46.131528 4011 services_controller.go:574] Full service sync requested for network=default W0327 15:03:46.132014 4011 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:03:46.244181 4011 default_network_controller.go:1008] Node "ip-10-0-0-80.ec2.internal" in remote zone "ip-10-0-0-80.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0327 15:03:46.244189 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-80.ec2.internal in network "default" I0327 15:03:46.244337 4011 node_tracker.go:169] Node ip-10-0-0-80.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:46.244354 4011 services_controller.go:574] Full service sync requested for network=default W0327 15:03:46.244651 4011 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:03:46.245061 4011 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-80.ec2.internal for the network default I0327 15:03:46.246152 4011 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-80.ec2.internal" on network "default" took: 1.108517ms W0327 15:03:47.595752 4011 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:03:48.798793 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:03:48.798811 4011 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-80.ec2.internal in network "default" I0327 15:03:48.798820 4011 egressfirewall.go:628] Syncing node "ip-10-0-0-80.ec2.internal" for egress firewall I0327 15:03:48.798820 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 33.579µs I0327 15:03:48.798848 4011 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-80.ec2.internal", IP: 10.0.0.80 to PMTUD blocking rules I0327 15:03:48.798866 4011 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-80.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:03:48.798871 4011 node_tracker.go:169] Node ip-10-0-0-80.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:48.798887 4011 services_controller.go:574] Full service sync requested for network=default W0327 15:03:48.799391 4011 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:03:48.805839 4011 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-80.ec2.internal. Updating no re-route policies I0327 15:03:48.814145 4011 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-80.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:03:48.814164 4011 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-80.ec2.internal", IP: 10.0.0.80 to PMTUD blocking rules I0327 15:03:52.244339 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:03:52.244376 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 42.904µs W0327 15:04:11.132327 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 15:04:11.132351 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 15:04:18.604470 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:04:32.295752 4011 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5f5595559-gf6sw] creating logical port openshift-ingress_router-default-5f5595559-gf6sw for pod on switch ip-10-0-2-39.ec2.internal I0327 15:04:32.295849 4011 kube.go:256] Updating pod openshift-ingress/router-default-5f5595559-gf6sw I0327 15:04:32.313304 4011 pod.go:62] [openshift-ingress/router-default-5f5595559-gf6sw] pod update took 17.481018ms I0327 15:04:32.313324 4011 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-5f5595559-gf6sw] addLogicalPort annotation time took 17.511067ms I0327 15:04:32.314380 4011 pods.go:271] [openshift-ingress/router-default-5f5595559-gf6sw] addLogicalPort took 18.643937ms, libovsdb time 786.982µs I0327 15:05:48.529174 4011 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:05:48.529792 4011 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:05:48.529820 4011 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:05:48.529829 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0327 15:05:48.529844 4011 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:05:48.529849 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0327 15:05:49.552213 4011 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:05:49.552242 4011 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:05:49.552256 4011 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:05:49.552269 4011 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:05:53.979285 4011 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 15:05:53.989368 4011 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 15:06:58.495314 4011 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x8j9v] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-555cc59f77-x8j9v for pod on switch ip-10-0-2-39.ec2.internal I0327 15:06:58.495430 4011 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x8j9v I0327 15:06:58.513755 4011 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x8j9v] pod update took 18.342405ms I0327 15:06:58.513775 4011 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x8j9v] addLogicalPort annotation time took 18.368157ms I0327 15:06:58.514697 4011 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x8j9v] addLogicalPort took 19.395156ms, libovsdb time 673.427µs I0327 15:06:58.852729 4011 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x8j9v c142ff15adcc28d0910700a0d2146dabac94703b4565de3c978a9e398218a13f network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x8j9v c142ff15adcc28d0910700a0d2146dabac94703b4565de3c978a9e398218a13f network default NAD default] I0327 15:06:58.861454 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-555cc59f77-x8j9v, hostIfaceName: c142ff15adcc28d, network: default, NAD default, SandboxID: "c142ff15adcc28d0910700a0d2146dabac94703b4565de3c978a9e398218a13f", PCI device ID: , UID: "5b450004-b3f2-46b9-aa90-4bef94d47354", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0327 15:06:58.900324 4011 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x8j9v c142ff15adcc28d0910700a0d2146dabac94703b4565de3c978a9e398218a13f network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-x8j9v c142ff15adcc28d0910700a0d2146dabac94703b4565de3c978a9e398218a13f network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:75:97:72:2b:9e\",\"name\":\"c142ff15adcc28d\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0d4a8917-e1ac-4067-8151-9c777742603c\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:07:02.628784 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 15:07:03.535837 4011 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0327 15:07:03.536686 4011 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 824.231µs I0327 15:07:03.558473 4011 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 15:07:04.320264 4011 namespace.go:98] [openshift-service-ca] adding namespace I0327 15:07:04.321129 4011 namespace.go:102] [openshift-service-ca] adding namespace took 844.224µs I0327 15:07:04.339793 4011 namespace.go:142] [openshift-service-ca] updating namespace W0327 15:07:10.594108 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 15:07:12.673895 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5877c497c6-7qwf2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:12.674462 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5877c497c6-7qwf2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:12.685075 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5877c497c6-7qwf2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:12.685096 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5877c497c6-7qwf2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:12.690640 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5877c497c6-7qwf2 I0327 15:07:12.690662 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5877c497c6-7qwf2 I0327 15:07:13.617944 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-66db7c9c95-jxct4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:13.618595 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-66db7c9c95-jxct4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:13.631906 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-66db7c9c95-jxct4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:13.631928 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-66db7c9c95-jxct4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:13.636606 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-66db7c9c95-jxct4 I0327 15:07:13.636625 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-66db7c9c95-jxct4 W0327 15:07:13.680241 4011 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:07:23.410336 4011 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-2sgdj ad1f6892a1c1d57a939b29b74199f8c180574bca6dd3446a1f31cec8e8d083c6 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-2sgdj ad1f6892a1c1d57a939b29b74199f8c180574bca6dd3446a1f31cec8e8d083c6 network default NAD default] I0327 15:07:23.411243 4011 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-xnhr4 ce7f856bf67c79f6bfbd53b7b08c0dc82920016ae066124dda77731c4d6c3da2 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-xnhr4 ce7f856bf67c79f6bfbd53b7b08c0dc82920016ae066124dda77731c4d6c3da2 network default NAD default] I0327 15:07:23.414872 4011 cni.go:403] [openshift-dns/dns-default-fhvsm 4835c20ae70b3cf208b1b2e9950e2f82f70944e102e3e75dd31a78cc684e97a8 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-fhvsm 4835c20ae70b3cf208b1b2e9950e2f82f70944e102e3e75dd31a78cc684e97a8 network default NAD default] I0327 15:07:23.415296 4011 cni.go:403] [openshift-image-registry/image-registry-5877c497c6-7vmg7 489b9b43fe517c8deece9375ad1bb720676009d0a98cf9912abae9e203fbe7f2 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-5877c497c6-7vmg7 489b9b43fe517c8deece9375ad1bb720676009d0a98cf9912abae9e203fbe7f2 network default NAD default] I0327 15:07:23.421492 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-2sgdj, hostIfaceName: ad1f6892a1c1d57, network: default, NAD default, SandboxID: "ad1f6892a1c1d57a939b29b74199f8c180574bca6dd3446a1f31cec8e8d083c6", PCI device ID: , UID: "f4b19a71-d70d-402b-b548-73919c4265ef", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0327 15:07:23.425583 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-xnhr4, hostIfaceName: ce7f856bf67c79f, network: default, NAD default, SandboxID: "ce7f856bf67c79f6bfbd53b7b08c0dc82920016ae066124dda77731c4d6c3da2", PCI device ID: , UID: "6296b3b0-78ad-4b64-9970-10d7231eedb9", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0327 15:07:23.427206 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-fhvsm, hostIfaceName: 4835c20ae70b3cf, network: default, NAD default, SandboxID: "4835c20ae70b3cf208b1b2e9950e2f82f70944e102e3e75dd31a78cc684e97a8", PCI device ID: , UID: "3c221502-7733-46e7-964d-cd8616ae549c", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0327 15:07:23.429271 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-5877c497c6-7vmg7, hostIfaceName: 489b9b43fe517c8, network: default, NAD default, SandboxID: "489b9b43fe517c8deece9375ad1bb720676009d0a98cf9912abae9e203fbe7f2", PCI device ID: , UID: "19c0d5ac-eb0f-408e-be1e-752f00b7015a", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0327 15:07:23.469668 4011 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-2sgdj ad1f6892a1c1d57a939b29b74199f8c180574bca6dd3446a1f31cec8e8d083c6 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-2sgdj ad1f6892a1c1d57a939b29b74199f8c180574bca6dd3446a1f31cec8e8d083c6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:a4:0d:15:0b:8f\",\"name\":\"ad1f6892a1c1d57\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3e1be8c9-620c-4f6c-8ea2-258b94663ab1\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:07:23.692217 4011 cni.go:424] [openshift-image-registry/image-registry-5877c497c6-7vmg7 489b9b43fe517c8deece9375ad1bb720676009d0a98cf9912abae9e203fbe7f2 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-5877c497c6-7vmg7 489b9b43fe517c8deece9375ad1bb720676009d0a98cf9912abae9e203fbe7f2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:b3:1e:61:1d:55\",\"name\":\"489b9b43fe517c8\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9fe39ae9-9dc2-4093-ae03-bb797cf4899b\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:07:23.692292 4011 cni.go:424] [openshift-dns/dns-default-fhvsm 4835c20ae70b3cf208b1b2e9950e2f82f70944e102e3e75dd31a78cc684e97a8 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-fhvsm 4835c20ae70b3cf208b1b2e9950e2f82f70944e102e3e75dd31a78cc684e97a8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:d0:80:bd:d7:71\",\"name\":\"4835c20ae70b3cf\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1ac487a1-5a0d-4357-a1ba-b1878ee1aea1\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:07:23.692298 4011 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-xnhr4 ce7f856bf67c79f6bfbd53b7b08c0dc82920016ae066124dda77731c4d6c3da2 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-xnhr4 ce7f856bf67c79f6bfbd53b7b08c0dc82920016ae066124dda77731c4d6c3da2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:22:b0:d8:ba:0a\",\"name\":\"ce7f856bf67c79f\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/502592cb-c895-4e4d-ad21-4773b1fa7f2e\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:07:32.067974 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 15:07:32.068001 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 15:07:32.072495 4011 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:07:32.077265 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:07:32.077285 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:07:32.077293 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:07:32.081390 4011 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0327 15:07:32.081404 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0327 15:07:32.081408 4011 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0327 15:07:32.199569 4011 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-kn852] creating logical port openshift-insights_insights-runtime-extractor-kn852 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:07:32.199672 4011 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-kn852 I0327 15:07:32.217977 4011 pod.go:62] [openshift-insights/insights-runtime-extractor-kn852] pod update took 18.333386ms I0327 15:07:32.217997 4011 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-kn852] addLogicalPort annotation time took 18.359833ms I0327 15:07:32.219004 4011 pods.go:271] [openshift-insights/insights-runtime-extractor-kn852] addLogicalPort took 19.467129ms, libovsdb time 737.696µs W0327 15:07:32.259576 4011 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:07:32.291588 4011 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:07:32.560909 4011 cni.go:403] [openshift-insights/insights-runtime-extractor-kn852 f93d0a406757a969f2eba495a838ce76997687d7490343bd4dee724ccc1389c7 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-kn852 f93d0a406757a969f2eba495a838ce76997687d7490343bd4dee724ccc1389c7 network default NAD default] I0327 15:07:32.569412 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-kn852, hostIfaceName: f93d0a406757a96, network: default, NAD default, SandboxID: "f93d0a406757a969f2eba495a838ce76997687d7490343bd4dee724ccc1389c7", PCI device ID: , UID: "cdb846b1-0537-4305-ad75-81b017fe1e8d", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0327 15:07:32.610069 4011 cni.go:424] [openshift-insights/insights-runtime-extractor-kn852 f93d0a406757a969f2eba495a838ce76997687d7490343bd4dee724ccc1389c7 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-kn852 f93d0a406757a969f2eba495a838ce76997687d7490343bd4dee724ccc1389c7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:78:75:06:d3:9d\",\"name\":\"f93d0a406757a96\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f81276a8-0960-4bcf-b3f2-6cb3f14e5f18\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:07:36.109599 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:07:36.109622 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 15:07:36.109628 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:07:36.768589 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0327 15:07:37.010408 4011 base_network_controller_pods.go:487] [default/openshift-console/console-8579887566-55wtk] creating logical port openshift-console_console-8579887566-55wtk for pod on switch ip-10-0-2-39.ec2.internal I0327 15:07:37.010487 4011 kube.go:256] Updating pod openshift-console/console-8579887566-55wtk I0327 15:07:37.030208 4011 pod.go:62] [openshift-console/console-8579887566-55wtk] pod update took 19.743944ms I0327 15:07:37.030226 4011 base_network_controller_pods.go:937] [default/openshift-console/console-8579887566-55wtk] addLogicalPort annotation time took 19.765066ms I0327 15:07:37.031262 4011 pods.go:271] [openshift-console/console-8579887566-55wtk] addLogicalPort took 20.865177ms, libovsdb time 773.363µs I0327 15:07:37.363722 4011 cni.go:403] [openshift-console/console-8579887566-55wtk 55d8514329dfd8557942e8016c175e2991826867321cd3b094c8e2c6a5298b1d network default NAD default] ADD starting CNI request [openshift-console/console-8579887566-55wtk 55d8514329dfd8557942e8016c175e2991826867321cd3b094c8e2c6a5298b1d network default NAD default] I0327 15:07:37.373104 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-8579887566-55wtk, hostIfaceName: 55d8514329dfd85, network: default, NAD default, SandboxID: "55d8514329dfd8557942e8016c175e2991826867321cd3b094c8e2c6a5298b1d", PCI device ID: , UID: "b52fff8d-0234-4c9f-bfe2-a9c37beda356", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0327 15:07:37.412791 4011 cni.go:424] [openshift-console/console-8579887566-55wtk 55d8514329dfd8557942e8016c175e2991826867321cd3b094c8e2c6a5298b1d network default NAD default] ADD finished CNI request [openshift-console/console-8579887566-55wtk 55d8514329dfd8557942e8016c175e2991826867321cd3b094c8e2c6a5298b1d network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:f9:d4:98:2d:c8\",\"name\":\"55d8514329dfd85\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1564ab2d-7dbc-4413-b287-94c5edd15a6e\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:07:41.706196 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 15:07:42.160091 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0327 15:07:44.851906 4011 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:07:44.852045 4011 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0327 15:07:44.875907 4011 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 23.896573ms I0327 15:07:44.875929 4011 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 23.928231ms I0327 15:07:44.877061 4011 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 25.166841ms, libovsdb time 803.335µs I0327 15:07:45.210525 4011 cni.go:403] [openshift-monitoring/alertmanager-main-1 624fc3565ca42eec54a255ffe7fa7f4bc5551ab18cdf309804b0e8e6134a5681 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 624fc3565ca42eec54a255ffe7fa7f4bc5551ab18cdf309804b0e8e6134a5681 network default NAD default] I0327 15:07:45.219600 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 624fc3565ca42ee, network: default, NAD default, SandboxID: "624fc3565ca42eec54a255ffe7fa7f4bc5551ab18cdf309804b0e8e6134a5681", PCI device ID: , UID: "a4eee37e-04b8-4785-8039-ab9a3bcd7664", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0327 15:07:45.262674 4011 cni.go:424] [openshift-monitoring/alertmanager-main-1 624fc3565ca42eec54a255ffe7fa7f4bc5551ab18cdf309804b0e8e6134a5681 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 624fc3565ca42eec54a255ffe7fa7f4bc5551ab18cdf309804b0e8e6134a5681 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:42:6b:6c:93:4d\",\"name\":\"624fc3565ca42ee\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e84b7475-c37f-4c2b-a618-40c625578112\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:07:45.641316 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:07:45.641338 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:07:46.800789 4011 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-869bc4dbf8-5vxc2] creating logical port openshift-monitoring_thanos-querier-869bc4dbf8-5vxc2 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:07:46.800891 4011 kube.go:256] Updating pod openshift-monitoring/thanos-querier-869bc4dbf8-5vxc2 I0327 15:07:46.823169 4011 pod.go:62] [openshift-monitoring/thanos-querier-869bc4dbf8-5vxc2] pod update took 22.316058ms I0327 15:07:46.823189 4011 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-869bc4dbf8-5vxc2] addLogicalPort annotation time took 22.339041ms I0327 15:07:46.824324 4011 pods.go:271] [openshift-monitoring/thanos-querier-869bc4dbf8-5vxc2] addLogicalPort took 23.551456ms, libovsdb time 823.64µs I0327 15:07:47.156978 4011 cni.go:403] [openshift-monitoring/thanos-querier-869bc4dbf8-5vxc2 ee3a25cb09a6199da3027b9e66070597369d9a825fbe904bfd7f7bb5ea4d2cde network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-869bc4dbf8-5vxc2 ee3a25cb09a6199da3027b9e66070597369d9a825fbe904bfd7f7bb5ea4d2cde network default NAD default] I0327 15:07:47.164737 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-869bc4dbf8-5vxc2, hostIfaceName: ee3a25cb09a6199, network: default, NAD default, SandboxID: "ee3a25cb09a6199da3027b9e66070597369d9a825fbe904bfd7f7bb5ea4d2cde", PCI device ID: , UID: "1fa632b2-bc1b-4815-8d2d-9aceee93bac5", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] W0327 15:07:47.198437 4011 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:47.198453 4011 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:47.198459 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 15:07:47.205848 4011 cni.go:424] [openshift-monitoring/thanos-querier-869bc4dbf8-5vxc2 ee3a25cb09a6199da3027b9e66070597369d9a825fbe904bfd7f7bb5ea4d2cde network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-869bc4dbf8-5vxc2 ee3a25cb09a6199da3027b9e66070597369d9a825fbe904bfd7f7bb5ea4d2cde network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:16:f7:b9:61:49\",\"name\":\"ee3a25cb09a6199\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6da853db-5000-41a9-a297-817f14d829b7\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:07:48.194193 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 15:07:48.407941 4011 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-jvnlj] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-jvnlj for pod on switch ip-10-0-2-39.ec2.internal I0327 15:07:48.408066 4011 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-jvnlj I0327 15:07:48.424364 4011 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jvnlj] pod update took 16.325076ms I0327 15:07:48.424389 4011 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-jvnlj] addLogicalPort annotation time took 16.354915ms I0327 15:07:48.425500 4011 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jvnlj] addLogicalPort took 17.575634ms, libovsdb time 754.705µs I0327 15:07:48.516866 4011 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-78c64d759d-6dpl9] creating logical port openshift-monitoring_metrics-server-78c64d759d-6dpl9 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:07:48.516963 4011 kube.go:256] Updating pod openshift-monitoring/metrics-server-78c64d759d-6dpl9 I0327 15:07:48.536139 4011 pod.go:62] [openshift-monitoring/metrics-server-78c64d759d-6dpl9] pod update took 19.199281ms I0327 15:07:48.536158 4011 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-78c64d759d-6dpl9] addLogicalPort annotation time took 19.224287ms I0327 15:07:48.537441 4011 pods.go:271] [openshift-monitoring/metrics-server-78c64d759d-6dpl9] addLogicalPort took 20.593763ms, libovsdb time 950.983µs W0327 15:07:48.715616 4011 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-659bf44899-rpc84 IP address from the namespace address-set, err: pod openshift-console/console-659bf44899-rpc84: no pod IPs found I0327 15:07:49.106991 4011 cni.go:403] [openshift-monitoring/metrics-server-78c64d759d-6dpl9 20f75d906785d65a86b30ab7dd1e0110532cddf7c17a8dc782eb2adfbf78f1ef network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-78c64d759d-6dpl9 20f75d906785d65a86b30ab7dd1e0110532cddf7c17a8dc782eb2adfbf78f1ef network default NAD default] I0327 15:07:49.116356 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-78c64d759d-6dpl9, hostIfaceName: 20f75d906785d65, network: default, NAD default, SandboxID: "20f75d906785d65a86b30ab7dd1e0110532cddf7c17a8dc782eb2adfbf78f1ef", PCI device ID: , UID: "28e59d82-456a-4b5a-b2cb-7ec9c348b022", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0327 15:07:49.160739 4011 cni.go:424] [openshift-monitoring/metrics-server-78c64d759d-6dpl9 20f75d906785d65a86b30ab7dd1e0110532cddf7c17a8dc782eb2adfbf78f1ef network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-78c64d759d-6dpl9 20f75d906785d65a86b30ab7dd1e0110532cddf7c17a8dc782eb2adfbf78f1ef network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:f2:32:11:30:05\",\"name\":\"20f75d906785d65\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/68fd8b0c-fadc-42f0-ad35-3ae64ef630ba\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:07:49.365283 4011 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jvnlj ff117421bd42914626b1dd363d73836cb7242ec931bbde519a1478f85a1576db network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jvnlj ff117421bd42914626b1dd363d73836cb7242ec931bbde519a1478f85a1576db network default NAD default] I0327 15:07:49.375344 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-jvnlj, hostIfaceName: ff117421bd42914, network: default, NAD default, SandboxID: "ff117421bd42914626b1dd363d73836cb7242ec931bbde519a1478f85a1576db", PCI device ID: , UID: "0d9fcba0-25b2-436e-8139-3a5b85452eb2", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0327 15:07:49.420428 4011 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jvnlj ff117421bd42914626b1dd363d73836cb7242ec931bbde519a1478f85a1576db network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jvnlj ff117421bd42914626b1dd363d73836cb7242ec931bbde519a1478f85a1576db network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:10:4c:ad:69:98\",\"name\":\"ff117421bd42914\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/77900e1c-3a57-4cb1-973b-71d8bd0c677d\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:07:50.112828 4011 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:07:50.112942 4011 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0327 15:07:50.141762 4011 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 28.854416ms I0327 15:07:50.141784 4011 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 28.88215ms I0327 15:07:50.142961 4011 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 30.140353ms, libovsdb time 865.759µs I0327 15:07:50.487436 4011 cni.go:403] [openshift-monitoring/prometheus-k8s-1 3df0385378f2d8aa45f62ed55462853ec1d6d2c5497e499d8c7502fe777e7579 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 3df0385378f2d8aa45f62ed55462853ec1d6d2c5497e499d8c7502fe777e7579 network default NAD default] I0327 15:07:50.496234 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 3df0385378f2d8a, network: default, NAD default, SandboxID: "3df0385378f2d8aa45f62ed55462853ec1d6d2c5497e499d8c7502fe777e7579", PCI device ID: , UID: "1cf13f12-81b4-47d4-ab14-75aaaa5e0f99", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0327 15:07:50.542651 4011 cni.go:424] [openshift-monitoring/prometheus-k8s-1 3df0385378f2d8aa45f62ed55462853ec1d6d2c5497e499d8c7502fe777e7579 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 3df0385378f2d8aa45f62ed55462853ec1d6d2c5497e499d8c7502fe777e7579 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:8d:fc:51:a9:45\",\"name\":\"3df0385378f2d8a\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fbe9cc46-1b07-4ffc-9a87-9fa853d59c4b\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:07:51.261688 4011 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:07:51.261710 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:07:51.752901 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0327 15:07:52.277325 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:07:52.309180 4011 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:52.309195 4011 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:52.309199 4011 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:52.309204 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:53.795495 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:07:53.837666 4011 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:07:53.837685 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:07:53.906873 4011 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:53.906894 4011 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:53.906901 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:55.882713 4011 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:55.882732 4011 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:55.882736 4011 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:55.882740 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 15:07:57.415213 4011 cni.go:403] [openshift-image-registry/image-registry-5877c497c6-7vmg7 489b9b43fe517c8deece9375ad1bb720676009d0a98cf9912abae9e203fbe7f2 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-5877c497c6-7vmg7 489b9b43fe517c8deece9375ad1bb720676009d0a98cf9912abae9e203fbe7f2 network default NAD default] I0327 15:07:57.492373 4011 cni.go:424] [openshift-image-registry/image-registry-5877c497c6-7vmg7 489b9b43fe517c8deece9375ad1bb720676009d0a98cf9912abae9e203fbe7f2 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-5877c497c6-7vmg7 489b9b43fe517c8deece9375ad1bb720676009d0a98cf9912abae9e203fbe7f2 network default NAD default], result "{}", err I0327 15:07:57.541853 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5877c497c6-7vmg7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:57.541937 4011 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5877c497c6-7vmg7 I0327 15:07:57.543213 4011 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5877c497c6-7vmg7, ips: 10.128.0.6 I0327 15:07:57.543248 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5877c497c6-7vmg7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:57.553061 4011 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-64dd4964c8-bc6xl] creating logical port openshift-image-registry_image-registry-64dd4964c8-bc6xl for pod on switch ip-10-0-2-39.ec2.internal I0327 15:07:57.553159 4011 kube.go:256] Updating pod openshift-image-registry/image-registry-64dd4964c8-bc6xl I0327 15:07:57.570441 4011 pod.go:62] [openshift-image-registry/image-registry-64dd4964c8-bc6xl] pod update took 17.307253ms I0327 15:07:57.570461 4011 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-64dd4964c8-bc6xl] addLogicalPort annotation time took 17.331234ms I0327 15:07:57.571489 4011 pods.go:271] [openshift-image-registry/image-registry-64dd4964c8-bc6xl] addLogicalPort took 18.453017ms, libovsdb time 719.142µs I0327 15:07:57.583746 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-65c4b7d4b8-946vl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:57.584092 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-65c4b7d4b8-946vl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:57.887050 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-65c4b7d4b8-946vl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:57.887073 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-65c4b7d4b8-946vl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:57.892008 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-65c4b7d4b8-946vl I0327 15:07:57.892035 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-65c4b7d4b8-946vl I0327 15:07:57.911617 4011 cni.go:403] [openshift-image-registry/image-registry-64dd4964c8-bc6xl 1624497eee99bfe5b5a1cf6a14930f83af5c70a5303c64f16ac3b264fb8fe795 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-64dd4964c8-bc6xl 1624497eee99bfe5b5a1cf6a14930f83af5c70a5303c64f16ac3b264fb8fe795 network default NAD default] I0327 15:07:57.920008 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-64dd4964c8-bc6xl, hostIfaceName: 1624497eee99bfe, network: default, NAD default, SandboxID: "1624497eee99bfe5b5a1cf6a14930f83af5c70a5303c64f16ac3b264fb8fe795", PCI device ID: , UID: "e74686f4-0f14-4155-8b07-675e0c855220", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0327 15:07:58.166487 4011 cni.go:424] [openshift-image-registry/image-registry-64dd4964c8-bc6xl 1624497eee99bfe5b5a1cf6a14930f83af5c70a5303c64f16ac3b264fb8fe795 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-64dd4964c8-bc6xl 1624497eee99bfe5b5a1cf6a14930f83af5c70a5303c64f16ac3b264fb8fe795 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:d6:d1:c3:5f:8e\",\"name\":\"1624497eee99bfe\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/356f2851-27b7-457c-9a9d-146f25116d99\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:07:58.247140 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:58.247160 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:58.247166 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:58.247171 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 15:07:58.262478 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5877c497c6-7vmg7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:58.262505 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5877c497c6-7vmg7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:58.265183 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5877c497c6-7vmg7 I0327 15:07:58.265234 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5877c497c6-7vmg7 I0327 15:08:07.991409 4011 obj_retry.go:464] Detected object openshift-console/console-8579887566-h47gd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:07.992010 4011 obj_retry.go:464] Detected object openshift-console/console-8579887566-h47gd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:08.856852 4011 obj_retry.go:458] Detected object openshift-console/console-8579887566-h47gd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:08.856874 4011 obj_retry.go:458] Detected object openshift-console/console-8579887566-h47gd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:08.861914 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8579887566-h47gd I0327 15:08:08.861932 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8579887566-h47gd I0327 15:08:12.365874 4011 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=5 W0327 15:08:13.240196 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:08:15.228281 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:08:15.228300 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:08:15.228306 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:08:15.228316 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:08:15.228321 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:08:15.228325 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:08:15.267951 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:08:15.267973 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:08:15.267980 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 15:08:17.130928 4011 obj_retry.go:464] Detected object openshift-console/console-659bf44899-57xvt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:17.131484 4011 obj_retry.go:464] Detected object openshift-console/console-659bf44899-57xvt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 15:08:17.988375 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0327 15:08:18.016779 4011 obj_retry.go:458] Detected object openshift-console/console-659bf44899-57xvt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:18.016800 4011 obj_retry.go:458] Detected object openshift-console/console-659bf44899-57xvt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:18.022001 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-659bf44899-57xvt I0327 15:08:18.022019 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-659bf44899-57xvt W0327 15:08:19.919979 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:08:19.920011 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:08:19.959519 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:08:19.959559 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:08:20.280177 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:08:27.918145 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 15:08:28.027554 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0327 15:08:33.976523 4011 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0327 15:08:44.086296 4011 cni.go:403] [openshift-console/console-8579887566-55wtk 55d8514329dfd8557942e8016c175e2991826867321cd3b094c8e2c6a5298b1d network default NAD default] DEL starting CNI request [openshift-console/console-8579887566-55wtk 55d8514329dfd8557942e8016c175e2991826867321cd3b094c8e2c6a5298b1d network default NAD default] I0327 15:08:44.177352 4011 cni.go:424] [openshift-console/console-8579887566-55wtk 55d8514329dfd8557942e8016c175e2991826867321cd3b094c8e2c6a5298b1d network default NAD default] DEL finished CNI request [openshift-console/console-8579887566-55wtk 55d8514329dfd8557942e8016c175e2991826867321cd3b094c8e2c6a5298b1d network default NAD default], result "{}", err I0327 15:08:44.235430 4011 obj_retry.go:464] Detected object openshift-console/console-8579887566-55wtk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:44.235572 4011 pods.go:176] Deleting pod: openshift-console/console-8579887566-55wtk I0327 15:08:44.238772 4011 pods.go:236] Attempting to release IPs for pod: openshift-console/console-8579887566-55wtk, ips: 10.128.0.16 I0327 15:08:44.238859 4011 obj_retry.go:464] Detected object openshift-console/console-8579887566-55wtk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:44.441819 4011 obj_retry.go:458] Detected object openshift-console/console-8579887566-55wtk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:44.441845 4011 obj_retry.go:458] Detected object openshift-console/console-8579887566-55wtk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:44.447403 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8579887566-55wtk I0327 15:08:44.447426 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8579887566-55wtk I0327 15:08:45.659586 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=5 I0327 15:08:46.283888 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-65c4b7d4b8-78rvv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:46.284436 4011 obj_retry.go:464] Detected object openshift-image-registry/image-registry-65c4b7d4b8-78rvv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:46.986917 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-65c4b7d4b8-78rvv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:46.986940 4011 obj_retry.go:458] Detected object openshift-image-registry/image-registry-65c4b7d4b8-78rvv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:46.991792 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-65c4b7d4b8-78rvv I0327 15:08:46.991814 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-65c4b7d4b8-78rvv I0327 15:08:50.505976 4011 cni.go:403] [openshift-multus/network-metrics-daemon-r9wb8 57578f652e8fe4b15c4807c817540e1d0a1d56514c296554784eb26a8db7366f network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-r9wb8 57578f652e8fe4b15c4807c817540e1d0a1d56514c296554784eb26a8db7366f network default NAD default] I0327 15:08:50.514490 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-r9wb8, hostIfaceName: 57578f652e8fe4b, network: default, NAD default, SandboxID: "57578f652e8fe4b15c4807c817540e1d0a1d56514c296554784eb26a8db7366f", PCI device ID: , UID: "7892dc99-45ed-4923-9329-dd813e5fc5a7", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] W0327 15:08:50.554252 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:08:50.554329 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 15:08:50.556721 4011 cni.go:424] [openshift-multus/network-metrics-daemon-r9wb8 57578f652e8fe4b15c4807c817540e1d0a1d56514c296554784eb26a8db7366f network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-r9wb8 57578f652e8fe4b15c4807c817540e1d0a1d56514c296554784eb26a8db7366f network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:ea:9b:be:29:73\",\"name\":\"57578f652e8fe4b\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b6e1068b-0fce-4a57-ac52-078d2c7f744f\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:08:56.870852 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=5 I0327 15:09:05.536043 4011 cni.go:403] [openshift-monitoring/alertmanager-main-1 624fc3565ca42eec54a255ffe7fa7f4bc5551ab18cdf309804b0e8e6134a5681 network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 624fc3565ca42eec54a255ffe7fa7f4bc5551ab18cdf309804b0e8e6134a5681 network default NAD default] I0327 15:09:05.611056 4011 cni.go:424] [openshift-monitoring/alertmanager-main-1 624fc3565ca42eec54a255ffe7fa7f4bc5551ab18cdf309804b0e8e6134a5681 network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 624fc3565ca42eec54a255ffe7fa7f4bc5551ab18cdf309804b0e8e6134a5681 network default NAD default], result "{}", err I0327 15:09:05.669687 4011 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:05.669714 4011 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0327 15:09:05.670978 4011 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.128.0.17 I0327 15:09:05.671007 4011 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:06.548595 4011 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:06.548623 4011 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:06.554888 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0327 15:09:06.554911 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0327 15:09:06.594295 4011 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:09:06.594392 4011 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0327 15:09:06.615598 4011 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 21.243465ms I0327 15:09:06.615617 4011 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 21.267089ms I0327 15:09:06.616882 4011 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 22.601109ms, libovsdb time 941.808µs I0327 15:09:06.950128 4011 cni.go:403] [openshift-monitoring/alertmanager-main-1 39edd70311b4fda37e5c8e3a3370a1e6400fabf1021fd1419dcf61327ca2e689 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 39edd70311b4fda37e5c8e3a3370a1e6400fabf1021fd1419dcf61327ca2e689 network default NAD default] I0327 15:09:06.958825 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 39edd70311b4fda, network: default, NAD default, SandboxID: "39edd70311b4fda37e5c8e3a3370a1e6400fabf1021fd1419dcf61327ca2e689", PCI device ID: , UID: "421e12ee-b97e-42fc-a27e-7f819750f79e", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0327 15:09:07.000089 4011 cni.go:424] [openshift-monitoring/alertmanager-main-1 39edd70311b4fda37e5c8e3a3370a1e6400fabf1021fd1419dcf61327ca2e689 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 39edd70311b4fda37e5c8e3a3370a1e6400fabf1021fd1419dcf61327ca2e689 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:f6:11:21:fc:60\",\"name\":\"39edd70311b4fda\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/71c4b23e-626d-4982-b1fb-1f0f0a11ebdc\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:09:09.834861 4011 cni.go:403] [openshift-monitoring/prometheus-k8s-1 3df0385378f2d8aa45f62ed55462853ec1d6d2c5497e499d8c7502fe777e7579 network default NAD default] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 3df0385378f2d8aa45f62ed55462853ec1d6d2c5497e499d8c7502fe777e7579 network default NAD default] I0327 15:09:09.910796 4011 cni.go:424] [openshift-monitoring/prometheus-k8s-1 3df0385378f2d8aa45f62ed55462853ec1d6d2c5497e499d8c7502fe777e7579 network default NAD default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 3df0385378f2d8aa45f62ed55462853ec1d6d2c5497e499d8c7502fe777e7579 network default NAD default], result "{}", err I0327 15:09:09.981842 4011 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:09.981873 4011 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0327 15:09:09.983123 4011 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.128.0.21 I0327 15:09:09.983155 4011 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:10.578128 4011 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:10.578152 4011 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:10.585233 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0327 15:09:10.585263 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0327 15:09:10.655175 4011 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:09:10.655289 4011 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0327 15:09:10.685287 4011 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 30.039204ms I0327 15:09:10.685309 4011 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 30.06485ms I0327 15:09:10.686577 4011 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 31.417489ms, libovsdb time 938.231µs I0327 15:09:11.015176 4011 cni.go:403] [openshift-monitoring/prometheus-k8s-1 18caed62c5702aa087ac668b04ff7aaeeeff0998dfddd98258d33295ff3df140 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 18caed62c5702aa087ac668b04ff7aaeeeff0998dfddd98258d33295ff3df140 network default NAD default] I0327 15:09:11.024700 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 18caed62c5702aa, network: default, NAD default, SandboxID: "18caed62c5702aa087ac668b04ff7aaeeeff0998dfddd98258d33295ff3df140", PCI device ID: , UID: "675cae3f-a6ec-4c36-9fee-106290ede5f2", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0327 15:09:11.065027 4011 cni.go:424] [openshift-monitoring/prometheus-k8s-1 18caed62c5702aa087ac668b04ff7aaeeeff0998dfddd98258d33295ff3df140 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 18caed62c5702aa087ac668b04ff7aaeeeff0998dfddd98258d33295ff3df140 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:d4:fa:53:41:a7\",\"name\":\"18caed62c5702aa\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/26bdab16-aa88-4dd6-b372-8178d63d6f4a\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:09:12.469841 4011 base_network_controller_pods.go:487] [default/openshift-console/console-6b65cb69c9-6xjhr] creating logical port openshift-console_console-6b65cb69c9-6xjhr for pod on switch ip-10-0-2-39.ec2.internal I0327 15:09:12.469936 4011 kube.go:256] Updating pod openshift-console/console-6b65cb69c9-6xjhr I0327 15:09:12.491385 4011 pod.go:62] [openshift-console/console-6b65cb69c9-6xjhr] pod update took 21.479307ms I0327 15:09:12.491407 4011 base_network_controller_pods.go:937] [default/openshift-console/console-6b65cb69c9-6xjhr] addLogicalPort annotation time took 21.504172ms I0327 15:09:12.492690 4011 pods.go:271] [openshift-console/console-6b65cb69c9-6xjhr] addLogicalPort took 22.862849ms, libovsdb time 828.599µs I0327 15:09:12.849463 4011 cni.go:403] [openshift-console/console-6b65cb69c9-6xjhr 2dbaf15c7e32f379bc2646ae48553482b58ac708555ba76196a74e38b4d6caa5 network default NAD default] ADD starting CNI request [openshift-console/console-6b65cb69c9-6xjhr 2dbaf15c7e32f379bc2646ae48553482b58ac708555ba76196a74e38b4d6caa5 network default NAD default] I0327 15:09:12.860997 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6b65cb69c9-6xjhr, hostIfaceName: 2dbaf15c7e32f37, network: default, NAD default, SandboxID: "2dbaf15c7e32f379bc2646ae48553482b58ac708555ba76196a74e38b4d6caa5", PCI device ID: , UID: "52a7bb2d-c8ae-4af4-89d0-d45eae79d81d", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0327 15:09:12.908779 4011 cni.go:424] [openshift-console/console-6b65cb69c9-6xjhr 2dbaf15c7e32f379bc2646ae48553482b58ac708555ba76196a74e38b4d6caa5 network default NAD default] ADD finished CNI request [openshift-console/console-6b65cb69c9-6xjhr 2dbaf15c7e32f379bc2646ae48553482b58ac708555ba76196a74e38b4d6caa5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:10:9e:60:2f:b2\",\"name\":\"2dbaf15c7e32f37\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e0b1e214-f8e6-4d25-a3e0-60135023fb44\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:09:17.243270 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 W0327 15:09:23.654682 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 15:09:37.011592 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:09:37.011614 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:09:37.011620 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 15:09:37.764402 4011 obj_retry.go:464] Detected object openshift-console/console-6865b86f9c-xttgq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:37.764959 4011 obj_retry.go:464] Detected object openshift-console/console-6865b86f9c-xttgq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:38.311059 4011 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:38.311648 4011 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:38.355938 4011 obj_retry.go:458] Detected object openshift-console/console-6865b86f9c-xttgq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.355953 4011 obj_retry.go:458] Detected object openshift-console/console-6865b86f9c-xttgq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.365739 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6865b86f9c-xttgq I0327 15:09:38.365759 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6865b86f9c-xttgq I0327 15:09:38.638265 4011 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.638315 4011 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.644354 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0327 15:09:38.644380 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0327 15:09:42.250625 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=650 I0327 15:09:48.969898 4011 obj_retry.go:464] Detected object openshift-console/console-6865b86f9c-hl52k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:48.970443 4011 obj_retry.go:464] Detected object openshift-console/console-6865b86f9c-hl52k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:49.214757 4011 obj_retry.go:458] Detected object openshift-console/console-6865b86f9c-hl52k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:49.214775 4011 obj_retry.go:458] Detected object openshift-console/console-6865b86f9c-hl52k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:49.218159 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6865b86f9c-hl52k I0327 15:09:49.218181 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6865b86f9c-hl52k I0327 15:10:00.222860 4011 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd 043bc572f638652965f437dd45f347c5714fc5cdd8409048e1176e001d19834c network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd 043bc572f638652965f437dd45f347c5714fc5cdd8409048e1176e001d19834c network default NAD default] I0327 15:10:00.299783 4011 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd 043bc572f638652965f437dd45f347c5714fc5cdd8409048e1176e001d19834c network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd 043bc572f638652965f437dd45f347c5714fc5cdd8409048e1176e001d19834c network default NAD default], result "{}", err I0327 15:10:00.806664 4011 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:00.806703 4011 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd I0327 15:10:00.808004 4011 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd, ips: 10.128.0.9 I0327 15:10:00.808032 4011 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:00.820317 4011 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:00.820337 4011 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:00.823807 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd I0327 15:10:00.823835 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd I0327 15:10:08.241325 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0327 15:10:10.238269 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=84 W0327 15:10:11.875704 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:10:11.875723 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 15:10:13.163743 4011 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:13.164298 4011 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:13.619754 4011 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:13.619785 4011 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:13.627206 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0327 15:10:13.627230 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0327 15:10:55.446870 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0327 15:10:55.608437 4011 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:10:55.608462 4011 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 30.493µs I0327 15:10:55.608506 4011 egressfirewall.go:628] Syncing node "ip-10-0-0-80.ec2.internal" for egress firewall I0327 15:10:57.247000 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 W0327 15:10:58.692521 4011 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:11:09.731859 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0327 15:11:09.731931 4011 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4cnll IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4cnll: no pod IPs found I0327 15:11:47.053343 4011 cni.go:403] [openshift-console/console-6b65cb69c9-6xjhr 2dbaf15c7e32f379bc2646ae48553482b58ac708555ba76196a74e38b4d6caa5 network default NAD default] DEL starting CNI request [openshift-console/console-6b65cb69c9-6xjhr 2dbaf15c7e32f379bc2646ae48553482b58ac708555ba76196a74e38b4d6caa5 network default NAD default] I0327 15:11:47.122965 4011 cni.go:424] [openshift-console/console-6b65cb69c9-6xjhr 2dbaf15c7e32f379bc2646ae48553482b58ac708555ba76196a74e38b4d6caa5 network default NAD default] DEL finished CNI request [openshift-console/console-6b65cb69c9-6xjhr 2dbaf15c7e32f379bc2646ae48553482b58ac708555ba76196a74e38b4d6caa5 network default NAD default], result "{}", err I0327 15:11:47.546114 4011 cni.go:403] [openshift-console/console-5bf9956b89-rslvj a9b00d6646d39a6246d67a9f22a07a3f3e1f5a60234ee3730db311b6da662344 network default NAD default] ADD starting CNI request [openshift-console/console-5bf9956b89-rslvj a9b00d6646d39a6246d67a9f22a07a3f3e1f5a60234ee3730db311b6da662344 network default NAD default] I0327 15:11:54.732503 4011 reflector.go:568] "Warning: watch ended with error" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732502 4011 reflector.go:568] "Warning: watch ended with error" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732524 4011 reflector.go:568] "Warning: watch ended with error" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732554 4011 reflector.go:568] "Warning: watch ended with error" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732555 4011 reflector.go:568] "Warning: watch ended with error" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732563 4011 reflector.go:568] "Warning: watch ended with error" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732577 4011 reflector.go:568] "Warning: watch ended with error" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732502 4011 reflector.go:568] "Warning: watch ended with error" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732589 4011 reflector.go:568] "Warning: watch ended with error" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732589 4011 reflector.go:568] "Warning: watch ended with error" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732597 4011 reflector.go:568] "Warning: watch ended with error" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732613 4011 reflector.go:568] "Warning: watch ended with error" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732611 4011 reflector.go:568] "Warning: watch ended with error" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732617 4011 reflector.go:568] "Warning: watch ended with error" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732629 4011 reflector.go:568] "Warning: watch ended with error" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:54.732660 4011 reflector.go:568] "Warning: watch ended with error" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" err="an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding" I0327 15:11:55.642785 4011 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:55.661402 4011 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:55.680654 4011 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:55.684509 4011 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:55.728884 4011 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:55.731841 4011 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:55.732869 4011 with_retry.go:234] "Got a Retry-After response" delay="1s" attempt=1 url="https://api.kx-c3e8d4b4b8.hypershift.local:443/api/v1/namespaces/openshift-console/pods/console-5bf9956b89-rslvj" I0327 15:11:55.784112 4011 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:55.787209 4011 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:55.811040 4011 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:55.814040 4011 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:55.818801 4011 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:55.823052 4011 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:55.929947 4011 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:55.933646 4011 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:55.939597 4011 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:11:55.943066 4011 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:11:55.967925 4011 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:55.970813 4011 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:56.002511 4011 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:56.006932 4011 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:56.007128 4011 namespace.go:142] [dedicated-admin] updating namespace I0327 15:11:56.007238 4011 namespace.go:142] [default] updating namespace I0327 15:11:56.007279 4011 namespace.go:142] [kube-node-lease] updating namespace I0327 15:11:56.007421 4011 namespace.go:142] [kube-public] updating namespace I0327 15:11:56.007506 4011 namespace.go:142] [kube-system] updating namespace I0327 15:11:56.007603 4011 namespace.go:142] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] updating namespace I0327 15:11:56.007708 4011 namespace.go:142] [open-cluster-management-agent-addon] updating namespace I0327 15:11:56.007805 4011 namespace.go:142] [openshift] updating namespace I0327 15:11:56.007873 4011 namespace.go:142] [openshift-apiserver] updating namespace I0327 15:11:56.008014 4011 namespace.go:142] [openshift-apiserver-operator] updating namespace I0327 15:11:56.008137 4011 namespace.go:142] [openshift-authentication-operator] updating namespace I0327 15:11:56.008149 4011 namespace.go:142] [openshift-authentication] updating namespace I0327 15:11:56.008280 4011 namespace.go:142] [openshift-backplane] updating namespace I0327 15:11:56.008325 4011 namespace.go:142] [openshift-backplane-cee] updating namespace I0327 15:11:56.008448 4011 namespace.go:142] [openshift-backplane-cse] updating namespace I0327 15:11:56.008571 4011 namespace.go:142] [openshift-backplane-csm] updating namespace I0327 15:11:56.008696 4011 namespace.go:142] [openshift-backplane-lpsre] updating namespace I0327 15:11:56.008741 4011 namespace.go:142] [openshift-backplane-managed-scripts] updating namespace I0327 15:11:56.008835 4011 namespace.go:142] [openshift-backplane-mcs-tier-two] updating namespace I0327 15:11:56.008923 4011 namespace.go:142] [openshift-backplane-mobb] updating namespace I0327 15:11:56.008964 4011 namespace.go:142] [openshift-backplane-srep] updating namespace I0327 15:11:56.009119 4011 namespace.go:142] [openshift-backplane-srep-ro] updating namespace I0327 15:11:56.009153 4011 namespace.go:142] [openshift-backplane-tam] updating namespace I0327 15:11:56.009259 4011 namespace.go:142] [openshift-cloud-controller-manager] updating namespace I0327 15:11:56.009408 4011 namespace.go:142] [openshift-cloud-credential-operator] updating namespace I0327 15:11:56.009447 4011 namespace.go:142] [openshift-cloud-network-config-controller] updating namespace I0327 15:11:56.009535 4011 namespace.go:142] [openshift-cluster-csi-drivers] updating namespace I0327 15:11:56.009656 4011 namespace.go:142] [openshift-cluster-machine-approver] updating namespace I0327 15:11:56.009762 4011 namespace.go:142] [openshift-cluster-node-tuning-operator] updating namespace I0327 15:11:56.009839 4011 namespace.go:142] [openshift-cluster-samples-operator] updating namespace I0327 15:11:56.009864 4011 namespace.go:142] [openshift-cluster-storage-operator] updating namespace I0327 15:11:56.009945 4011 namespace.go:142] [openshift-cluster-version] updating namespace I0327 15:11:56.010062 4011 namespace.go:142] [openshift-config] updating namespace I0327 15:11:56.010220 4011 namespace.go:142] [openshift-config-managed] updating namespace I0327 15:11:56.010315 4011 namespace.go:142] [openshift-config-operator] updating namespace I0327 15:11:56.010341 4011 namespace.go:142] [openshift-console] updating namespace I0327 15:11:56.010433 4011 namespace.go:142] [openshift-console-operator] updating namespace I0327 15:11:56.010481 4011 namespace.go:142] [openshift-console-user-settings] updating namespace I0327 15:11:56.010660 4011 namespace.go:142] [openshift-controller-manager] updating namespace I0327 15:11:56.010682 4011 namespace.go:142] [openshift-controller-manager-operator] updating namespace I0327 15:11:56.010808 4011 namespace.go:142] [openshift-customer-monitoring] updating namespace I0327 15:11:56.010914 4011 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 15:11:56.011101 4011 namespace.go:142] [openshift-dns] updating namespace I0327 15:11:56.011105 4011 namespace.go:142] [openshift-dns-operator] updating namespace I0327 15:11:56.011191 4011 namespace.go:142] [openshift-etcd] updating namespace I0327 15:11:56.011366 4011 namespace.go:142] [openshift-host-network] updating namespace I0327 15:11:56.011396 4011 namespace.go:142] [openshift-image-registry] updating namespace I0327 15:11:56.011452 4011 namespace.go:142] [openshift-infra] updating namespace I0327 15:11:56.011574 4011 namespace.go:142] [openshift-ingress] updating namespace I0327 15:11:56.011686 4011 namespace.go:142] [openshift-ingress-canary] updating namespace I0327 15:11:56.011775 4011 namespace.go:142] [openshift-ingress-operator] updating namespace I0327 15:11:56.011874 4011 namespace.go:142] [openshift-insights] updating namespace I0327 15:11:56.011997 4011 namespace.go:142] [openshift-kube-apiserver] updating namespace I0327 15:11:56.012102 4011 namespace.go:142] [openshift-kube-apiserver-operator] updating namespace I0327 15:11:56.012155 4011 namespace.go:142] [openshift-kube-controller-manager] updating namespace I0327 15:11:56.012259 4011 namespace.go:142] [openshift-kube-controller-manager-operator] updating namespace I0327 15:11:56.012378 4011 namespace.go:142] [openshift-kube-scheduler] updating namespace I0327 15:11:56.012485 4011 namespace.go:142] [openshift-kube-scheduler-operator] updating namespace I0327 15:11:56.012610 4011 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 15:11:56.012646 4011 namespace.go:142] [openshift-kube-storage-version-migrator-operator] updating namespace I0327 15:11:56.012755 4011 namespace.go:142] [openshift-logging] updating namespace I0327 15:11:56.012765 4011 namespace.go:142] [openshift-machine-api] updating namespace I0327 15:11:56.012876 4011 namespace.go:142] [openshift-machine-config-operator] updating namespace I0327 15:11:56.013022 4011 namespace.go:142] [openshift-marketplace] updating namespace I0327 15:11:56.013142 4011 namespace.go:142] [openshift-monitoring] updating namespace I0327 15:11:56.013162 4011 namespace.go:142] [openshift-multus] updating namespace I0327 15:11:56.013306 4011 namespace.go:142] [openshift-must-gather-operator] updating namespace I0327 15:11:56.013417 4011 namespace.go:142] [openshift-network-console] updating namespace I0327 15:11:56.013505 4011 namespace.go:142] [openshift-network-diagnostics] updating namespace I0327 15:11:56.013595 4011 namespace.go:142] [openshift-network-node-identity] updating namespace I0327 15:11:56.013730 4011 namespace.go:142] [openshift-network-operator] updating namespace I0327 15:11:56.013758 4011 namespace.go:142] [openshift-node] updating namespace I0327 15:11:56.013845 4011 namespace.go:142] [openshift-operator-lifecycle-manager] updating namespace I0327 15:11:56.013919 4011 namespace.go:142] [openshift-operators] updating namespace I0327 15:11:56.013991 4011 namespace.go:142] [openshift-operators-redhat] updating namespace I0327 15:11:56.014148 4011 namespace.go:142] [openshift-ovn-kubernetes] updating namespace I0327 15:11:56.014182 4011 namespace.go:142] [openshift-package-operator] updating namespace I0327 15:11:56.014293 4011 namespace.go:142] [openshift-route-controller-manager] updating namespace I0327 15:11:56.014394 4011 namespace.go:142] [openshift-service-ca] updating namespace I0327 15:11:56.014439 4011 namespace.go:142] [openshift-service-ca-operator] updating namespace I0327 15:11:56.014582 4011 namespace.go:142] [openshift-user-workload-monitoring] updating namespace I0327 15:11:56.136095 4011 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:11:56.139340 4011 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:11:56.173378 4011 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:56.174458 4011 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:56.176686 4011 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:56.195232 4011 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:11:56.195442 4011 pods.go:176] Deleting pod: openshift-console/console-6b65cb69c9-6xjhr E0327 15:11:56.195750 4011 egressqos.go:875] "Unhandled Error" err=< couldn't get key for object {Key:openshift-console/console-6b65cb69c9-6xjhr Obj:&Pod{ObjectMeta:{console-6b65cb69c9-6xjhr console-6b65cb69c9- openshift-console 52a7bb2d-c8ae-4af4-89d0-d45eae79d81d 14438 1 2026-03-27 15:09:12 +0000 UTC map[app:console component:ui pod-template-hash:6b65cb69c9] map[console.openshift.io/authn-ca-trust-config-version:11265 console.openshift.io/console-config-version:14312 console.openshift.io/image:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51 console.openshift.io/infrastructure-config-version:1452 console.openshift.io/oauth-secret-version:11292 console.openshift.io/proxy-config-version:1461 console.openshift.io/service-ca-config-version:11787 console.openshift.io/trusted-ca-config-version:11824 k8s.ovn.org/pod-networks:{"default":{"ip_addresses":["10.128.0.25/23"],"mac_address":"0a:58:0a:80:00:19","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/14","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.25/23","gateway_ip":"10.128.0.1","role":"primary"}} k8s.v1.cni.cncf.io/network-status:[{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.25" ], "mac": "0a:58:0a:80:00:19", "default": true, "dns": {} }] openshift.io/required-scc:restricted-v2 openshift.io/scc:restricted-v2 seccomp.security.alpha.kubernetes.io/pod:runtime/default security.openshift.io/validated-scc-subject-type:user] [{apps/v1 ReplicaSet console-6b65cb69c9 b1992b3c-bc87-45d0-b71c-3d7dc42b9d5f 0xc00b8e40e7 0xc00b8e40e8}] [] []},Spec:PodSpec{Volumes:[]Volume{},Containers:[]Container{Container{Name:console,Image:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:https,HostPort:0,ContainerPort:8443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{10 -3} {} 10m DecimalSI},memory: {{104857600 0} {} 100Mi BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{},LivenessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/health,Port:{0 8443 },Host:,Scheme:HTTPS,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:10,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:1,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/health,Port:{0 8443 },Host:,Scheme:HTTPS,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[sleep 25],},HTTPGet:nil,TCPSocket:nil,Sleep:nil,},StopSignal:nil,},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:*1000210000,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*false,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,AppArmorProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:FallbackToLogsOnError,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/health,Port:{0 8443 },Host:,Scheme:HTTPS,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:10,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:30,TerminationGracePeriodSeconds:nil,},ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,RestartPolicyRules:[]ContainerRestartRule{},},},RestartPolicy:Always,TerminationGracePeriodSeconds:*40,ActiveDeadlineSeconds:nil,DNSPolicy:ClusterFirst,NodeSelector:map[string]string{},ServiceAccountName:console,DeprecatedServiceAccount:console,NodeName:ip-10-0-2-39.ec2.internal,HostNetwork:false,HostPID:false,HostIPC:false,SecurityContext:&PodSecurityContext{SELinuxOptions:&SELinuxOptions{User:,Role:,Type:,Level:s0:c15,c0,},RunAsUser:nil,RunAsNonRoot:*true,SupplementalGroups:[],FSGroup:*1000210000,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:&SeccompProfile{Type:RuntimeDefault,LocalhostProfile:nil,},AppArmorProfile:nil,SupplementalGroupsPolicy:nil,SELinuxChangePolicy:nil,},ImagePullSecrets:[]LocalObjectReference{LocalObjectReference{Name:console-dockercfg-kffqc,},},Hostname:,Subdomain:,Affinity:&Affinity{NodeAffinity:nil,PodAffinity:nil,PodAntiAffinity:&PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution:[]PodAffinityTerm{PodAffinityTerm{LabelSelector:&v1.LabelSelector{MatchLabels:map[string]string{},MatchExpressions:[]LabelSelectorRequirement{LabelSelectorRequirement{Key:component,Operator:In,Values:[ui],},},},Namespaces:[],TopologyKey:kubernetes.io/hostname,NamespaceSelector:nil,MatchLabelKeys:[],MismatchLabelKeys:[],},},PreferredDuringSchedulingIgnoredDuringExecution:[]WeightedPodAffinityTerm{},},},SchedulerName:default-scheduler,InitContainers:[]Container{},AutomountServiceAccountToken:nil,Tolerations:[]Toleration{Toleration{Key:node-role.kubernetes.io/master,Operator:Exists,Value:,Effect:NoSchedule,TolerationSeconds:nil,},Toleration{Key:node.kubernetes.io/unreachable,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*120,},Toleration{Key:node.kubernetes.io/not-ready,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},Toleration{Key:node.kubernetes.io/memory-pressure,Operator:Exists,Value:,Effect:NoSchedule,TolerationSeconds:nil,},},HostAliases:[]HostAlias{},PriorityClassName:system-cluster-critical,Priority:*2000000000,DNSConfig:nil,ShareProcessNamespace:nil,ReadinessGates:[]PodReadinessGate{},RuntimeClassName:nil,EnableServiceLinks:*true,PreemptionPolicy:*PreemptLowerPriority,Overhead:ResourceList{},TopologySpreadConstraints:[]TopologySpreadConstraint{},EphemeralContainers:[]EphemeralContainer{},SetHostnameAsFQDN:nil,OS:nil,HostUsers:nil,SchedulingGates:[]PodSchedulingGate{},ResourceClaims:[]PodResourceClaim{},Resources:nil,HostnameOverride:nil,},Status:PodStatus{Phase:Running,Conditions:[]PodCondition{PodCondition{Type:PodReadyToStartContainers,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:13 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:Initialized,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:12 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:Ready,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:23 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:ContainersReady,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:23 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:PodScheduled,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:12 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},},Message:,Reason:,HostIP:10.0.2.39,PodIP:10.128.0.25,StartTime:2026-03-27 15:09:12 +0000 UTC,ContainerStatuses:[]ContainerStatus{ContainerStatus{Name:console,State:ContainerState{Waiting:nil,Running:&ContainerStateRunning{StartedAt:2026-03-27 15:09:13 +0000 UTC,},Terminated:nil,},LastTerminationState:ContainerState{Waiting:nil,Running:nil,Terminated:nil,},Ready:true,RestartCount:0,Image:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51,ImageID:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51,ContainerID:cri-o://192fc0fb5ec93f26cee462a642e27a6140b6b4b6fd2ce7bb249165b08aefcb9a,Started:*true,AllocatedResources:ResourceList{cpu: {{10 -3} {} 10m DecimalSI},memory: {{104857600 0} {} 100Mi BinarySI},},Resources:&ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{10 -3} {} 10m DecimalSI},memory: {{104857600 0} {} 100Mi BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMountStatus{VolumeMountStatus{Name:console-serving-cert,MountPath:/var/serving-cert,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:console-oauth-config,MountPath:/var/oauth-config,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:console-config,MountPath:/var/console-config,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:service-ca,MountPath:/var/service-ca,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:trusted-ca-bundle,MountPath:/etc/pki/ca-trust/extracted/pem,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:oauth-serving-cert,MountPath:/var/oauth-serving-cert,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:kube-api-access-s8tlm,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,ReadOnly:true,RecursiveReadOnly:*Disabled,},},User:&ContainerUser{Linux:&LinuxContainerUser{UID:1000210000,GID:0,SupplementalGroups:[0 1000210000],},},AllocatedResourcesStatus:[]ResourceStatus{},StopSignal:nil,},},QOSClass:Burstable,InitContainerStatuses:[]ContainerStatus{},NominatedNodeName:,PodIPs:[]PodIP{PodIP{IP:10.128.0.25,},},EphemeralContainerStatuses:[]ContainerStatus{},Resize:,ResourceClaimStatuses:[]PodResourceClaimStatus{},HostIPs:[]HostIP{HostIP{IP:10.0.2.39,},},ObservedGeneration:1,ExtendedResourceClaimStatus:nil,},}}: object has no meta: object does not implement the Object interfaces > logger="UnhandledError" E0327 15:11:56.195755 4011 admin_network_policy_controller.go:544] "Unhandled Error" err=< couldn't get key for object {Key:openshift-console/console-6b65cb69c9-6xjhr Obj:&Pod{ObjectMeta:{console-6b65cb69c9-6xjhr console-6b65cb69c9- openshift-console 52a7bb2d-c8ae-4af4-89d0-d45eae79d81d 14438 1 2026-03-27 15:09:12 +0000 UTC map[app:console component:ui pod-template-hash:6b65cb69c9] map[console.openshift.io/authn-ca-trust-config-version:11265 console.openshift.io/console-config-version:14312 console.openshift.io/image:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51 console.openshift.io/infrastructure-config-version:1452 console.openshift.io/oauth-secret-version:11292 console.openshift.io/proxy-config-version:1461 console.openshift.io/service-ca-config-version:11787 console.openshift.io/trusted-ca-config-version:11824 k8s.ovn.org/pod-networks:{"default":{"ip_addresses":["10.128.0.25/23"],"mac_address":"0a:58:0a:80:00:19","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/14","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.25/23","gateway_ip":"10.128.0.1","role":"primary"}} k8s.v1.cni.cncf.io/network-status:[{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.25" ], "mac": "0a:58:0a:80:00:19", "default": true, "dns": {} }] openshift.io/required-scc:restricted-v2 openshift.io/scc:restricted-v2 seccomp.security.alpha.kubernetes.io/pod:runtime/default security.openshift.io/validated-scc-subject-type:user] [{apps/v1 ReplicaSet console-6b65cb69c9 b1992b3c-bc87-45d0-b71c-3d7dc42b9d5f 0xc00b8e40e7 0xc00b8e40e8}] [] []},Spec:PodSpec{Volumes:[]Volume{},Containers:[]Container{Container{Name:console,Image:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:https,HostPort:0,ContainerPort:8443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{10 -3} {} 10m DecimalSI},memory: {{104857600 0} {} 100Mi BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{},LivenessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/health,Port:{0 8443 },Host:,Scheme:HTTPS,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:10,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:1,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/health,Port:{0 8443 },Host:,Scheme:HTTPS,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[sleep 25],},HTTPGet:nil,TCPSocket:nil,Sleep:nil,},StopSignal:nil,},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:*1000210000,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*false,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,AppArmorProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:FallbackToLogsOnError,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/health,Port:{0 8443 },Host:,Scheme:HTTPS,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:10,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:30,TerminationGracePeriodSeconds:nil,},ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,RestartPolicyRules:[]ContainerRestartRule{},},},RestartPolicy:Always,TerminationGracePeriodSeconds:*40,ActiveDeadlineSeconds:nil,DNSPolicy:ClusterFirst,NodeSelector:map[string]string{},ServiceAccountName:console,DeprecatedServiceAccount:console,NodeName:ip-10-0-2-39.ec2.internal,HostNetwork:false,HostPID:false,HostIPC:false,SecurityContext:&PodSecurityContext{SELinuxOptions:&SELinuxOptions{User:,Role:,Type:,Level:s0:c15,c0,},RunAsUser:nil,RunAsNonRoot:*true,SupplementalGroups:[],FSGroup:*1000210000,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:&SeccompProfile{Type:RuntimeDefault,LocalhostProfile:nil,},AppArmorProfile:nil,SupplementalGroupsPolicy:nil,SELinuxChangePolicy:nil,},ImagePullSecrets:[]LocalObjectReference{LocalObjectReference{Name:console-dockercfg-kffqc,},},Hostname:,Subdomain:,Affinity:&Affinity{NodeAffinity:nil,PodAffinity:nil,PodAntiAffinity:&PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution:[]PodAffinityTerm{PodAffinityTerm{LabelSelector:&v1.LabelSelector{MatchLabels:map[string]string{},MatchExpressions:[]LabelSelectorRequirement{LabelSelectorRequirement{Key:component,Operator:In,Values:[ui],},},},Namespaces:[],TopologyKey:kubernetes.io/hostname,NamespaceSelector:nil,MatchLabelKeys:[],MismatchLabelKeys:[],},},PreferredDuringSchedulingIgnoredDuringExecution:[]WeightedPodAffinityTerm{},},},SchedulerName:default-scheduler,InitContainers:[]Container{},AutomountServiceAccountToken:nil,Tolerations:[]Toleration{Toleration{Key:node-role.kubernetes.io/master,Operator:Exists,Value:,Effect:NoSchedule,TolerationSeconds:nil,},Toleration{Key:node.kubernetes.io/unreachable,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*120,},Toleration{Key:node.kubernetes.io/not-ready,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},Toleration{Key:node.kubernetes.io/memory-pressure,Operator:Exists,Value:,Effect:NoSchedule,TolerationSeconds:nil,},},HostAliases:[]HostAlias{},PriorityClassName:system-cluster-critical,Priority:*2000000000,DNSConfig:nil,ShareProcessNamespace:nil,ReadinessGates:[]PodReadinessGate{},RuntimeClassName:nil,EnableServiceLinks:*true,PreemptionPolicy:*PreemptLowerPriority,Overhead:ResourceList{},TopologySpreadConstraints:[]TopologySpreadConstraint{},EphemeralContainers:[]EphemeralContainer{},SetHostnameAsFQDN:nil,OS:nil,HostUsers:nil,SchedulingGates:[]PodSchedulingGate{},ResourceClaims:[]PodResourceClaim{},Resources:nil,HostnameOverride:nil,},Status:PodStatus{Phase:Running,Conditions:[]PodCondition{PodCondition{Type:PodReadyToStartContainers,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:13 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:Initialized,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:12 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:Ready,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:23 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:ContainersReady,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:23 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:PodScheduled,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:12 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},},Message:,Reason:,HostIP:10.0.2.39,PodIP:10.128.0.25,StartTime:2026-03-27 15:09:12 +0000 UTC,ContainerStatuses:[]ContainerStatus{ContainerStatus{Name:console,State:ContainerState{Waiting:nil,Running:&ContainerStateRunning{StartedAt:2026-03-27 15:09:13 +0000 UTC,},Terminated:nil,},LastTerminationState:ContainerState{Waiting:nil,Running:nil,Terminated:nil,},Ready:true,RestartCount:0,Image:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51,ImageID:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51,ContainerID:cri-o://192fc0fb5ec93f26cee462a642e27a6140b6b4b6fd2ce7bb249165b08aefcb9a,Started:*true,AllocatedResources:ResourceList{cpu: {{10 -3} {} 10m DecimalSI},memory: {{104857600 0} {} 100Mi BinarySI},},Resources:&ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{10 -3} {} 10m DecimalSI},memory: {{104857600 0} {} 100Mi BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMountStatus{VolumeMountStatus{Name:console-serving-cert,MountPath:/var/serving-cert,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:console-oauth-config,MountPath:/var/oauth-config,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:console-config,MountPath:/var/console-config,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:service-ca,MountPath:/var/service-ca,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:trusted-ca-bundle,MountPath:/etc/pki/ca-trust/extracted/pem,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:oauth-serving-cert,MountPath:/var/oauth-serving-cert,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:kube-api-access-s8tlm,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,ReadOnly:true,RecursiveReadOnly:*Disabled,},},User:&ContainerUser{Linux:&LinuxContainerUser{UID:1000210000,GID:0,SupplementalGroups:[0 1000210000],},},AllocatedResourcesStatus:[]ResourceStatus{},StopSignal:nil,},},QOSClass:Burstable,InitContainerStatuses:[]ContainerStatus{},NominatedNodeName:,PodIPs:[]PodIP{PodIP{IP:10.128.0.25,},},EphemeralContainerStatuses:[]ContainerStatus{},Resize:,ResourceClaimStatuses:[]PodResourceClaimStatus{},HostIPs:[]HostIP{HostIP{IP:10.0.2.39,},},ObservedGeneration:1,ExtendedResourceClaimStatus:nil,},}}: object has no meta: object does not implement the Object interfaces > logger="UnhandledError" E0327 15:11:56.195756 4011 controller.go:236] "Unhandled Error" err=< controller udn-host-isolation-manager: couldn't get key for object {Key:openshift-console/console-6b65cb69c9-6xjhr Obj:&Pod{ObjectMeta:{console-6b65cb69c9-6xjhr console-6b65cb69c9- openshift-console 52a7bb2d-c8ae-4af4-89d0-d45eae79d81d 14438 1 2026-03-27 15:09:12 +0000 UTC map[app:console component:ui pod-template-hash:6b65cb69c9] map[console.openshift.io/authn-ca-trust-config-version:11265 console.openshift.io/console-config-version:14312 console.openshift.io/image:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51 console.openshift.io/infrastructure-config-version:1452 console.openshift.io/oauth-secret-version:11292 console.openshift.io/proxy-config-version:1461 console.openshift.io/service-ca-config-version:11787 console.openshift.io/trusted-ca-config-version:11824 k8s.ovn.org/pod-networks:{"default":{"ip_addresses":["10.128.0.25/23"],"mac_address":"0a:58:0a:80:00:19","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/14","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.25/23","gateway_ip":"10.128.0.1","role":"primary"}} k8s.v1.cni.cncf.io/network-status:[{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.25" ], "mac": "0a:58:0a:80:00:19", "default": true, "dns": {} }] openshift.io/required-scc:restricted-v2 openshift.io/scc:restricted-v2 seccomp.security.alpha.kubernetes.io/pod:runtime/default security.openshift.io/validated-scc-subject-type:user] [{apps/v1 ReplicaSet console-6b65cb69c9 b1992b3c-bc87-45d0-b71c-3d7dc42b9d5f 0xc00b8e40e7 0xc00b8e40e8}] [] []},Spec:PodSpec{Volumes:[]Volume{},Containers:[]Container{Container{Name:console,Image:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51,Command:[],Args:[],WorkingDir:,Ports:[]ContainerPort{ContainerPort{Name:https,HostPort:0,ContainerPort:8443,Protocol:TCP,HostIP:,},},Env:[]EnvVar{},Resources:ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{10 -3} {} 10m DecimalSI},memory: {{104857600 0} {} 100Mi BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMount{},LivenessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/health,Port:{0 8443 },Host:,Scheme:HTTPS,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:10,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:1,TerminationGracePeriodSeconds:nil,},ReadinessProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/health,Port:{0 8443 },Host:,Scheme:HTTPS,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:1,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:3,TerminationGracePeriodSeconds:nil,},Lifecycle:&Lifecycle{PostStart:nil,PreStop:&LifecycleHandler{Exec:&ExecAction{Command:[sleep 25],},HTTPGet:nil,TCPSocket:nil,Sleep:nil,},StopSignal:nil,},TerminationMessagePath:/dev/termination-log,ImagePullPolicy:IfNotPresent,SecurityContext:&SecurityContext{Capabilities:&Capabilities{Add:[],Drop:[ALL],},Privileged:nil,SELinuxOptions:nil,RunAsUser:*1000210000,RunAsNonRoot:nil,ReadOnlyRootFilesystem:*false,AllowPrivilegeEscalation:*false,RunAsGroup:nil,ProcMount:nil,WindowsOptions:nil,SeccompProfile:nil,AppArmorProfile:nil,},Stdin:false,StdinOnce:false,TTY:false,EnvFrom:[]EnvFromSource{},TerminationMessagePolicy:FallbackToLogsOnError,VolumeDevices:[]VolumeDevice{},StartupProbe:&Probe{ProbeHandler:ProbeHandler{Exec:nil,HTTPGet:&HTTPGetAction{Path:/health,Port:{0 8443 },Host:,Scheme:HTTPS,HTTPHeaders:[]HTTPHeader{},},TCPSocket:nil,GRPC:nil,},InitialDelaySeconds:0,TimeoutSeconds:10,PeriodSeconds:10,SuccessThreshold:1,FailureThreshold:30,TerminationGracePeriodSeconds:nil,},ResizePolicy:[]ContainerResizePolicy{},RestartPolicy:nil,RestartPolicyRules:[]ContainerRestartRule{},},},RestartPolicy:Always,TerminationGracePeriodSeconds:*40,ActiveDeadlineSeconds:nil,DNSPolicy:ClusterFirst,NodeSelector:map[string]string{},ServiceAccountName:console,DeprecatedServiceAccount:console,NodeName:ip-10-0-2-39.ec2.internal,HostNetwork:false,HostPID:false,HostIPC:false,SecurityContext:&PodSecurityContext{SELinuxOptions:&SELinuxOptions{User:,Role:,Type:,Level:s0:c15,c0,},RunAsUser:nil,RunAsNonRoot:*true,SupplementalGroups:[],FSGroup:*1000210000,RunAsGroup:nil,Sysctls:[]Sysctl{},WindowsOptions:nil,FSGroupChangePolicy:nil,SeccompProfile:&SeccompProfile{Type:RuntimeDefault,LocalhostProfile:nil,},AppArmorProfile:nil,SupplementalGroupsPolicy:nil,SELinuxChangePolicy:nil,},ImagePullSecrets:[]LocalObjectReference{LocalObjectReference{Name:console-dockercfg-kffqc,},},Hostname:,Subdomain:,Affinity:&Affinity{NodeAffinity:nil,PodAffinity:nil,PodAntiAffinity:&PodAntiAffinity{RequiredDuringSchedulingIgnoredDuringExecution:[]PodAffinityTerm{PodAffinityTerm{LabelSelector:&v1.LabelSelector{MatchLabels:map[string]string{},MatchExpressions:[]LabelSelectorRequirement{LabelSelectorRequirement{Key:component,Operator:In,Values:[ui],},},},Namespaces:[],TopologyKey:kubernetes.io/hostname,NamespaceSelector:nil,MatchLabelKeys:[],MismatchLabelKeys:[],},},PreferredDuringSchedulingIgnoredDuringExecution:[]WeightedPodAffinityTerm{},},},SchedulerName:default-scheduler,InitContainers:[]Container{},AutomountServiceAccountToken:nil,Tolerations:[]Toleration{Toleration{Key:node-role.kubernetes.io/master,Operator:Exists,Value:,Effect:NoSchedule,TolerationSeconds:nil,},Toleration{Key:node.kubernetes.io/unreachable,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*120,},Toleration{Key:node.kubernetes.io/not-ready,Operator:Exists,Value:,Effect:NoExecute,TolerationSeconds:*300,},Toleration{Key:node.kubernetes.io/memory-pressure,Operator:Exists,Value:,Effect:NoSchedule,TolerationSeconds:nil,},},HostAliases:[]HostAlias{},PriorityClassName:system-cluster-critical,Priority:*2000000000,DNSConfig:nil,ShareProcessNamespace:nil,ReadinessGates:[]PodReadinessGate{},RuntimeClassName:nil,EnableServiceLinks:*true,PreemptionPolicy:*PreemptLowerPriority,Overhead:ResourceList{},TopologySpreadConstraints:[]TopologySpreadConstraint{},EphemeralContainers:[]EphemeralContainer{},SetHostnameAsFQDN:nil,OS:nil,HostUsers:nil,SchedulingGates:[]PodSchedulingGate{},ResourceClaims:[]PodResourceClaim{},Resources:nil,HostnameOverride:nil,},Status:PodStatus{Phase:Running,Conditions:[]PodCondition{PodCondition{Type:PodReadyToStartContainers,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:13 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:Initialized,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:12 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:Ready,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:23 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:ContainersReady,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:23 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},PodCondition{Type:PodScheduled,Status:True,LastProbeTime:0001-01-01 00:00:00 +0000 UTC,LastTransitionTime:2026-03-27 15:09:12 +0000 UTC,Reason:,Message:,ObservedGeneration:1,},},Message:,Reason:,HostIP:10.0.2.39,PodIP:10.128.0.25,StartTime:2026-03-27 15:09:12 +0000 UTC,ContainerStatuses:[]ContainerStatus{ContainerStatus{Name:console,State:ContainerState{Waiting:nil,Running:&ContainerStateRunning{StartedAt:2026-03-27 15:09:13 +0000 UTC,},Terminated:nil,},LastTerminationState:ContainerState{Waiting:nil,Running:nil,Terminated:nil,},Ready:true,RestartCount:0,Image:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51,ImageID:quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:07a6c0a310270b830b96fcb8fca9cd11e45569276335bb5334b18b388d6eed51,ContainerID:cri-o://192fc0fb5ec93f26cee462a642e27a6140b6b4b6fd2ce7bb249165b08aefcb9a,Started:*true,AllocatedResources:ResourceList{cpu: {{10 -3} {} 10m DecimalSI},memory: {{104857600 0} {} 100Mi BinarySI},},Resources:&ResourceRequirements{Limits:ResourceList{},Requests:ResourceList{cpu: {{10 -3} {} 10m DecimalSI},memory: {{104857600 0} {} 100Mi BinarySI},},Claims:[]ResourceClaim{},},VolumeMounts:[]VolumeMountStatus{VolumeMountStatus{Name:console-serving-cert,MountPath:/var/serving-cert,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:console-oauth-config,MountPath:/var/oauth-config,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:console-config,MountPath:/var/console-config,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:service-ca,MountPath:/var/service-ca,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:trusted-ca-bundle,MountPath:/etc/pki/ca-trust/extracted/pem,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:oauth-serving-cert,MountPath:/var/oauth-serving-cert,ReadOnly:true,RecursiveReadOnly:*Disabled,},VolumeMountStatus{Name:kube-api-access-s8tlm,MountPath:/var/run/secrets/kubernetes.io/serviceaccount,ReadOnly:true,RecursiveReadOnly:*Disabled,},},User:&ContainerUser{Linux:&LinuxContainerUser{UID:1000210000,GID:0,SupplementalGroups:[0 1000210000],},},AllocatedResourcesStatus:[]ResourceStatus{},StopSignal:nil,},},QOSClass:Burstable,InitContainerStatuses:[]ContainerStatus{},NominatedNodeName:,PodIPs:[]PodIP{PodIP{IP:10.128.0.25,},},EphemeralContainerStatuses:[]ContainerStatus{},Resize:,ResourceClaimStatuses:[]PodResourceClaimStatus{},HostIPs:[]HostIP{HostIP{IP:10.0.2.39,},},ObservedGeneration:1,ExtendedResourceClaimStatus:nil,},}}: object has no meta: object does not implement the Object interfaces > logger="UnhandledError" I0327 15:11:56.196856 4011 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6b65cb69c9-6xjhr, ips: 10.128.0.25 I0327 15:11:56.197664 4011 base_network_controller_pods.go:487] [default/openshift-console/console-5bf9956b89-rslvj] creating logical port openshift-console_console-5bf9956b89-rslvj for pod on switch ip-10-0-2-39.ec2.internal I0327 15:11:56.197780 4011 kube.go:256] Updating pod openshift-console/console-5bf9956b89-rslvj I0327 15:11:56.198044 4011 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:56.198064 4011 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:56.198080 4011 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:56.198093 4011 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:56.212912 4011 pod.go:62] [openshift-console/console-5bf9956b89-rslvj] pod update took 15.156474ms I0327 15:11:56.212933 4011 base_network_controller_pods.go:937] [default/openshift-console/console-5bf9956b89-rslvj] addLogicalPort annotation time took 15.180185ms I0327 15:11:56.213984 4011 pods.go:271] [openshift-console/console-5bf9956b89-rslvj] addLogicalPort took 16.329337ms, libovsdb time 705.046µs I0327 15:11:56.228421 4011 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:56.231234 4011 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:56.274095 4011 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:11:56.276975 4011 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:11:56.298714 4011 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:56.301604 4011 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:11:56.356875 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5bf9956b89-rslvj, hostIfaceName: a9b00d6646d39a6, network: default, NAD default, SandboxID: "a9b00d6646d39a6246d67a9f22a07a3f3e1f5a60234ee3730db311b6da662344", PCI device ID: , UID: "5535669d-c93e-4055-a455-5c664930fd5b", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0327 15:11:56.398179 4011 cni.go:424] [openshift-console/console-5bf9956b89-rslvj a9b00d6646d39a6246d67a9f22a07a3f3e1f5a60234ee3730db311b6da662344 network default NAD default] ADD finished CNI request [openshift-console/console-5bf9956b89-rslvj a9b00d6646d39a6246d67a9f22a07a3f3e1f5a60234ee3730db311b6da662344 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:22:3b:d2:3f:44\",\"name\":\"a9b00d6646d39a6\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b761ca0f-4608-49be-ae18-51747dbf0a65\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:11:57.918803 4011 obj_retry.go:464] Detected object openshift-console/console-6b65cb69c9-pgrnb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:57.919352 4011 obj_retry.go:464] Detected object openshift-console/console-6b65cb69c9-pgrnb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 15:11:58.258786 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 15:11:58.268907 4011 obj_retry.go:458] Detected object openshift-console/console-6b65cb69c9-pgrnb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:58.268929 4011 obj_retry.go:458] Detected object openshift-console/console-6b65cb69c9-pgrnb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:58.278196 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6b65cb69c9-pgrnb I0327 15:11:58.278213 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6b65cb69c9-pgrnb I0327 15:16:36.847585 4011 namespace.go:98] [tsf] adding namespace I0327 15:16:36.848492 4011 namespace.go:102] [tsf] adding namespace took 881.1µs I0327 15:16:36.866149 4011 namespace.go:142] [tsf] updating namespace I0327 15:16:36.880848 4011 namespace.go:142] [tsf] updating namespace I0327 15:16:40.388320 4011 namespace.go:98] [cert-manager-operator] adding namespace I0327 15:16:40.389247 4011 namespace.go:102] [cert-manager-operator] adding namespace took 900.145µs I0327 15:16:40.392802 4011 namespace.go:98] [rhbk-operator] adding namespace I0327 15:16:40.392920 4011 namespace.go:98] [konflux-ui] adding namespace I0327 15:16:40.393013 4011 namespace.go:98] [rhtpa-operator] adding namespace I0327 15:16:40.393142 4011 namespace.go:98] [tsf-tpa] adding namespace I0327 15:16:40.393614 4011 namespace.go:102] [rhbk-operator] adding namespace took 792.981µs I0327 15:16:40.394001 4011 namespace.go:98] [tsf-keycloak] adding namespace I0327 15:16:40.394027 4011 namespace.go:98] [tsf-tas] adding namespace I0327 15:16:40.394301 4011 namespace.go:102] [konflux-ui] adding namespace took 1.361019ms I0327 15:16:40.394330 4011 namespace.go:98] [konflux-operator] adding namespace I0327 15:16:40.394887 4011 namespace.go:102] [konflux-operator] adding namespace took 548.408µs I0327 15:16:40.395375 4011 namespace.go:102] [rhtpa-operator] adding namespace took 2.348067ms I0327 15:16:40.395977 4011 namespace.go:102] [tsf-tpa] adding namespace took 2.818449ms I0327 15:16:40.396423 4011 namespace.go:102] [tsf-keycloak] adding namespace took 2.408443ms I0327 15:16:40.396832 4011 namespace.go:102] [tsf-tas] adding namespace took 2.794906ms I0327 15:16:40.405469 4011 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:40.422963 4011 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:40.426985 4011 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:40.441746 4011 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:40.448991 4011 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:40.451901 4011 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:40.451948 4011 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:40.457808 4011 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:40.462918 4011 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:40.462940 4011 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:40.464623 4011 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:40.465993 4011 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:40.480740 4011 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:40.495116 4011 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:40.502414 4011 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:40.517094 4011 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:40.523212 4011 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:40.541590 4011 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:40.545518 4011 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:40.559355 4011 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:40.563706 4011 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:40.575409 4011 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:40.581203 4011 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:40.589785 4011 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:41.533263 4011 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:41.546184 4011 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:59.280203 4011 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0327 15:17:02.198335 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=38 I0327 15:17:04.459227 4011 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:04.459775 4011 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:04.459802 4011 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:04.459812 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:04.459832 4011 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:04.459839 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:05.482197 4011 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:05.482222 4011 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:05.482240 4011 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:05.482251 4011 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:06.482691 4011 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:06.483182 4011 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:06.483202 4011 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:06.483209 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:06.483218 4011 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:06.483222 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:07.504106 4011 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.504527 4011 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.504566 4011 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.504576 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:07.504589 4011 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.504593 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:07.516242 4011 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:07.516257 4011 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:07.516268 4011 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:07.516278 4011 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:07.526005 4011 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.526368 4011 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.526386 4011 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.526391 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:07.526403 4011 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.526411 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:08.527402 4011 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.527429 4011 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.527445 4011 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.527460 4011 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.559452 4011 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.559475 4011 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.559486 4011 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.559497 4011 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:09.519262 4011 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.519804 4011 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.519831 4011 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.519839 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:09.519851 4011 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.519856 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:09.550175 4011 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.550576 4011 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.550600 4011 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.550609 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:09.550625 4011 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.550632 4011 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:10.570784 4011 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.570809 4011 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.570829 4011 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.570848 4011 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.600073 4011 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.600095 4011 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.600108 4011 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.600120 4011 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:15.722481 4011 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-wjgmh] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-wjgmh for pod on switch ip-10-0-2-39.ec2.internal I0327 15:17:15.722672 4011 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-wjgmh I0327 15:17:15.746192 4011 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-wjgmh] pod update took 23.55343ms I0327 15:17:15.746221 4011 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-wjgmh] addLogicalPort annotation time took 23.588992ms I0327 15:17:15.747796 4011 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-wjgmh] addLogicalPort took 25.330822ms, libovsdb time 1.160387ms I0327 15:17:16.096387 4011 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-wjgmh 9b8c6a6000bcc8030b7bd36f832545bb6b4729a422d988dbd795dee28c2f6b12 network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-wjgmh 9b8c6a6000bcc8030b7bd36f832545bb6b4729a422d988dbd795dee28c2f6b12 network default NAD default] I0327 15:17:16.104501 4011 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-wjgmh, hostIfaceName: 9b8c6a6000bcc80, network: default, NAD default, SandboxID: "9b8c6a6000bcc8030b7bd36f832545bb6b4729a422d988dbd795dee28c2f6b12", PCI device ID: , UID: "a77c8449-6b93-4642-9bb4-cbf397b5275d", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0327 15:17:16.146771 4011 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-wjgmh 9b8c6a6000bcc8030b7bd36f832545bb6b4729a422d988dbd795dee28c2f6b12 network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-wjgmh 9b8c6a6000bcc8030b7bd36f832545bb6b4729a422d988dbd795dee28c2f6b12 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:26:70:7c:ce:f9\",\"name\":\"9b8c6a6000bcc80\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/566293a6-98ae-4dd5-b344-58116e726e9d\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:17:18.199624 4011 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-mht92] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-mht92 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:17:18.199792 4011 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 I0327 15:17:18.222355 4011 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92] pod update took 22.603505ms I0327 15:17:18.222381 4011 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-mht92] addLogicalPort annotation time took 22.633821ms I0327 15:17:18.223662 4011 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92] addLogicalPort took 24.061487ms, libovsdb time 893.865µs I0327 15:17:19.265947 4011 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 a332e5626de406b786b81e5a7076e2a5d329de123ee0ebd20bbe901884efba73 network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 a332e5626de406b786b81e5a7076e2a5d329de123ee0ebd20bbe901884efba73 network default NAD default] I0327 15:17:19.275242 4011 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-mht92, hostIfaceName: a332e5626de406b, network: default, NAD default, SandboxID: "a332e5626de406b786b81e5a7076e2a5d329de123ee0ebd20bbe901884efba73", PCI device ID: , UID: "9e8ba43a-692d-465a-b161-cad1d8062dba", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0327 15:17:19.319710 4011 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 a332e5626de406b786b81e5a7076e2a5d329de123ee0ebd20bbe901884efba73 network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 a332e5626de406b786b81e5a7076e2a5d329de123ee0ebd20bbe901884efba73 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:d5:e6:ba:7a:49\",\"name\":\"a332e5626de406b\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f3575b36-9bfd-46ac-9ad1-5e2e9cb6fac2\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:17:19.722491 4011 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-8468f77bc5-4srgv] creating logical port rhbk-operator_rhbk-operator-8468f77bc5-4srgv for pod on switch ip-10-0-2-39.ec2.internal I0327 15:17:19.722625 4011 kube.go:256] Updating pod rhbk-operator/rhbk-operator-8468f77bc5-4srgv I0327 15:17:19.742959 4011 pod.go:62] [rhbk-operator/rhbk-operator-8468f77bc5-4srgv] pod update took 20.362298ms I0327 15:17:19.742985 4011 base_network_controller_pods.go:937] [default/rhbk-operator/rhbk-operator-8468f77bc5-4srgv] addLogicalPort annotation time took 20.392278ms I0327 15:17:19.744197 4011 pods.go:271] [rhbk-operator/rhbk-operator-8468f77bc5-4srgv] addLogicalPort took 21.72777ms, libovsdb time 807.565µs I0327 15:17:20.097484 4011 cni.go:403] [rhbk-operator/rhbk-operator-8468f77bc5-4srgv ab4c9096d1d00f9d4c6a976b2aee619eb3d4b0cbfcfe7e9a855e36c9b6e83404 network default NAD default] ADD starting CNI request [rhbk-operator/rhbk-operator-8468f77bc5-4srgv ab4c9096d1d00f9d4c6a976b2aee619eb3d4b0cbfcfe7e9a855e36c9b6e83404 network default NAD default] I0327 15:17:20.106209 4011 helper_linux.go:450] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-8468f77bc5-4srgv, hostIfaceName: ab4c9096d1d00f9, network: default, NAD default, SandboxID: "ab4c9096d1d00f9d4c6a976b2aee619eb3d4b0cbfcfe7e9a855e36c9b6e83404", PCI device ID: , UID: "5f2cc0d3-b053-46e5-8add-0c5ce4b73772", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0327 15:17:20.147775 4011 cni.go:424] [rhbk-operator/rhbk-operator-8468f77bc5-4srgv ab4c9096d1d00f9d4c6a976b2aee619eb3d4b0cbfcfe7e9a855e36c9b6e83404 network default NAD default] ADD finished CNI request [rhbk-operator/rhbk-operator-8468f77bc5-4srgv ab4c9096d1d00f9d4c6a976b2aee619eb3d4b0cbfcfe7e9a855e36c9b6e83404 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:39:e7:f8:e2:ea\",\"name\":\"ab4c9096d1d00f9\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/059a21fc-3268-443a-91d1-87dc8b179ddf\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:17:20.946398 4011 namespace.go:98] [cert-manager] adding namespace I0327 15:17:20.947341 4011 namespace.go:102] [cert-manager] adding namespace took 921.672µs I0327 15:17:21.140191 4011 namespace.go:142] [cert-manager] updating namespace I0327 15:17:21.177074 4011 namespace.go:142] [cert-manager] updating namespace W0327 15:17:22.619972 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0327 15:17:23.492572 4011 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8rv2b] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-8rv2b for pod on switch ip-10-0-2-39.ec2.internal I0327 15:17:23.492709 4011 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8rv2b I0327 15:17:23.512441 4011 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8rv2b] pod update took 19.754384ms I0327 15:17:23.512465 4011 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8rv2b] addLogicalPort annotation time took 19.78419ms I0327 15:17:23.513893 4011 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8rv2b] addLogicalPort took 21.356112ms, libovsdb time 1.009446ms I0327 15:17:24.179774 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0327 15:17:25.132573 4011 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8rv2b dc0e2bf816e0d2a48a32c9e3e1d213f59ded2cf17502e0b0b6803a07e4cb200d network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8rv2b dc0e2bf816e0d2a48a32c9e3e1d213f59ded2cf17502e0b0b6803a07e4cb200d network default NAD default] I0327 15:17:25.141208 4011 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-8rv2b, hostIfaceName: dc0e2bf816e0d2a, network: default, NAD default, SandboxID: "dc0e2bf816e0d2a48a32c9e3e1d213f59ded2cf17502e0b0b6803a07e4cb200d", PCI device ID: , UID: "605d1d70-841a-4c32-823d-aec62b98f146", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0327 15:17:25.184912 4011 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8rv2b dc0e2bf816e0d2a48a32c9e3e1d213f59ded2cf17502e0b0b6803a07e4cb200d network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-8rv2b dc0e2bf816e0d2a48a32c9e3e1d213f59ded2cf17502e0b0b6803a07e4cb200d network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:42:49:b8:cd:d5\",\"name\":\"dc0e2bf816e0d2a\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/faba6be7-9c1b-4410-b07d-b667986df997\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:17:26.208144 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:17:26.213575 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:17:26.231003 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:17:26.257511 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:17:26.926471 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 15:17:26.926496 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 15:17:28.545281 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 15:17:29.400845 4011 namespace.go:142] [openshift-operators] updating namespace I0327 15:17:29.410460 4011 namespace.go:142] [openshift-operators] updating namespace W0327 15:17:33.525611 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0327 15:17:36.378929 4011 namespace.go:142] [openshift-operators] updating namespace I0327 15:17:36.911305 4011 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-tmdkl] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-tmdkl for pod on switch ip-10-0-2-39.ec2.internal I0327 15:17:36.911439 4011 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-tmdkl I0327 15:17:36.948215 4011 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-tmdkl] pod update took 36.803261ms I0327 15:17:36.948244 4011 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-tmdkl] addLogicalPort annotation time took 36.831504ms I0327 15:17:36.949425 4011 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-tmdkl] addLogicalPort took 38.143336ms, libovsdb time 786.464µs I0327 15:17:37.274329 4011 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-tmdkl 33bb882f316e961edfcf15dc3d8ecfc6703f6eff31a5025adafe8493917092f4 network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-tmdkl 33bb882f316e961edfcf15dc3d8ecfc6703f6eff31a5025adafe8493917092f4 network default NAD default] I0327 15:17:37.283084 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-tmdkl, hostIfaceName: 33bb882f316e961, network: default, NAD default, SandboxID: "33bb882f316e961edfcf15dc3d8ecfc6703f6eff31a5025adafe8493917092f4", PCI device ID: , UID: "a350c4f9-0059-4fa6-b545-093e62e9542e", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0327 15:17:37.326352 4011 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-tmdkl 33bb882f316e961edfcf15dc3d8ecfc6703f6eff31a5025adafe8493917092f4 network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-tmdkl 33bb882f316e961edfcf15dc3d8ecfc6703f6eff31a5025adafe8493917092f4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:b4:ff:d1:2b:e6\",\"name\":\"33bb882f316e961\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f478544-f58e-45af-b0d9-719ba249b171\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:17:39.144136 4011 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 W0327 15:17:39.672595 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 15:17:42.571109 4011 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-25hlf] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-25hlf for pod on switch ip-10-0-2-39.ec2.internal I0327 15:17:42.571220 4011 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-25hlf I0327 15:17:42.594705 4011 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-25hlf] pod update took 23.51353ms I0327 15:17:42.594724 4011 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-25hlf] addLogicalPort annotation time took 23.537876ms I0327 15:17:42.595957 4011 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-25hlf] addLogicalPort took 24.870155ms, libovsdb time 844.467µs I0327 15:17:42.939972 4011 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-25hlf 26dbad05e894807e94a542fb9564d76a309b69884f047df1aa942edb5fc763a6 network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-25hlf 26dbad05e894807e94a542fb9564d76a309b69884f047df1aa942edb5fc763a6 network default NAD default] I0327 15:17:42.949065 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-25hlf, hostIfaceName: 26dbad05e894807, network: default, NAD default, SandboxID: "26dbad05e894807e94a542fb9564d76a309b69884f047df1aa942edb5fc763a6", PCI device ID: , UID: "cd2d3ce7-523c-44b7-a476-3b8cb9e3a4b3", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0327 15:17:42.992820 4011 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-25hlf 26dbad05e894807e94a542fb9564d76a309b69884f047df1aa942edb5fc763a6 network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-25hlf 26dbad05e894807e94a542fb9564d76a309b69884f047df1aa942edb5fc763a6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:ce:4b:7e:38:85\",\"name\":\"26dbad05e894807\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/28e3119a-f842-4748-9c02-d404f8643bdc\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:17:45.912637 4011 namespace.go:98] [trusted-artifact-signer] adding namespace I0327 15:17:45.913516 4011 namespace.go:102] [trusted-artifact-signer] adding namespace took 856.872µs I0327 15:17:45.938686 4011 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:45.950679 4011 namespace.go:142] [trusted-artifact-signer] updating namespace W0327 15:17:46.663531 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 15:17:47.018022 4011 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-fswzc] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-fswzc for pod on switch ip-10-0-2-39.ec2.internal I0327 15:17:47.018123 4011 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-fswzc I0327 15:17:47.041795 4011 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-fswzc] pod update took 23.702953ms I0327 15:17:47.041816 4011 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-fswzc] addLogicalPort annotation time took 23.730847ms I0327 15:17:47.043111 4011 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-fswzc] addLogicalPort took 25.107334ms, libovsdb time 970.91µs I0327 15:17:47.377197 4011 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-fswzc 0f9f6c1b3398ad26958159bab9924634fd73231f40ae5fb72f9645e38d335bd3 network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-fswzc 0f9f6c1b3398ad26958159bab9924634fd73231f40ae5fb72f9645e38d335bd3 network default NAD default] I0327 15:17:47.386319 4011 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-fswzc, hostIfaceName: 0f9f6c1b3398ad2, network: default, NAD default, SandboxID: "0f9f6c1b3398ad26958159bab9924634fd73231f40ae5fb72f9645e38d335bd3", PCI device ID: , UID: "3f2f057d-b458-4a79-8838-7c0407623535", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0327 15:17:47.429777 4011 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-fswzc 0f9f6c1b3398ad26958159bab9924634fd73231f40ae5fb72f9645e38d335bd3 network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-fswzc 0f9f6c1b3398ad26958159bab9924634fd73231f40ae5fb72f9645e38d335bd3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:47:ba:df:d3:ab\",\"name\":\"0f9f6c1b3398ad2\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/76c93054-98ca-4d8f-b396-0e5634ea552b\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:17:50.759948 4011 namespace.go:98] [openshift-pipelines] adding namespace I0327 15:17:50.760856 4011 namespace.go:102] [openshift-pipelines] adding namespace took 881.977µs I0327 15:17:50.777670 4011 namespace.go:142] [openshift-pipelines] updating namespace I0327 15:17:51.471709 4011 namespace.go:142] [cert-manager] updating namespace I0327 15:17:51.671407 4011 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:52.075393 4011 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:52.274641 4011 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:52.476579 4011 namespace.go:142] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] updating namespace I0327 15:17:52.878867 4011 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:53.100079 4011 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:53.309064 4011 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:53.518105 4011 namespace.go:142] [tsf] updating namespace I0327 15:17:53.711195 4011 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:53.913455 4011 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:54.119264 4011 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:54.243305 4011 namespace.go:142] [cert-manager] updating namespace I0327 15:17:54.253203 4011 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:54.260077 4011 namespace.go:142] [default] updating namespace I0327 15:17:54.269746 4011 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:54.277217 4011 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:54.287231 4011 namespace.go:142] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] updating namespace I0327 15:17:54.299021 4011 namespace.go:142] [openshift] updating namespace I0327 15:17:54.306839 4011 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:54.318858 4011 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:54.325875 4011 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:54.335759 4011 namespace.go:142] [tsf] updating namespace I0327 15:17:54.352391 4011 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:54.372324 4011 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:54.394219 4011 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:54.493787 4011 namespace.go:142] [cert-manager] updating namespace I0327 15:17:54.595294 4011 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:54.694856 4011 namespace.go:142] [default] updating namespace I0327 15:17:54.793569 4011 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:54.899476 4011 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:54.998109 4011 namespace.go:142] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] updating namespace I0327 15:17:55.094219 4011 namespace.go:142] [openshift] updating namespace I0327 15:17:55.192432 4011 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:55.295244 4011 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:55.394738 4011 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:55.492099 4011 namespace.go:142] [tsf] updating namespace I0327 15:17:55.595237 4011 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:55.699777 4011 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:55.793227 4011 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:56.056195 4011 namespace.go:142] [openshift-pipelines] updating namespace W0327 15:17:57.756509 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 15:17:58.827583 4011 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:58.828087 4011 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:58.860460 4011 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:58.860489 4011 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:58.865426 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0327 15:17:58.865451 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0327 15:18:02.234718 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 15:18:02.790270 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0327 15:18:03.581796 4011 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-b79bdd59-jk5j8] creating logical port openshift-pipelines_tekton-events-controller-b79bdd59-jk5j8 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:18:03.581881 4011 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-b79bdd59-jk5j8 I0327 15:18:03.595928 4011 pod.go:62] [openshift-pipelines/tekton-events-controller-b79bdd59-jk5j8] pod update took 14.070278ms I0327 15:18:03.595951 4011 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-events-controller-b79bdd59-jk5j8] addLogicalPort annotation time took 14.097311ms I0327 15:18:03.597300 4011 pods.go:271] [openshift-pipelines/tekton-events-controller-b79bdd59-jk5j8] addLogicalPort took 15.523439ms, libovsdb time 968.445µs I0327 15:18:03.954836 4011 cni.go:403] [openshift-pipelines/tekton-events-controller-b79bdd59-jk5j8 ad3819c89f1367a47e13791d0f563cb015ba9285f358e5320475d3c2d7aca22f network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-jk5j8 ad3819c89f1367a47e13791d0f563cb015ba9285f358e5320475d3c2d7aca22f network default NAD default] I0327 15:18:03.964516 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-b79bdd59-jk5j8, hostIfaceName: ad3819c89f1367a, network: default, NAD default, SandboxID: "ad3819c89f1367a47e13791d0f563cb015ba9285f358e5320475d3c2d7aca22f", PCI device ID: , UID: "952881df-1fc2-4dd2-b8d7-d9e2f54af25e", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0327 15:18:04.011659 4011 cni.go:424] [openshift-pipelines/tekton-events-controller-b79bdd59-jk5j8 ad3819c89f1367a47e13791d0f563cb015ba9285f358e5320475d3c2d7aca22f network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-jk5j8 ad3819c89f1367a47e13791d0f563cb015ba9285f358e5320475d3c2d7aca22f network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:c7:a3:fa:ac:33\",\"name\":\"ad3819c89f1367a\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b2221ed5-c389-4fce-8516-db70f39b7924\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:18:06.782224 4011 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:18:06.782325 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:18:06.782377 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:18:07.795508 4011 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:18:07.795528 4011 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:18:07.795532 4011 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:18:08.122259 4011 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:18:08.122284 4011 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:18:08.122291 4011 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:18:10.792912 4011 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:18:10.792930 4011 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:18:10.792935 4011 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:18:10.792939 4011 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:18:11.797811 4011 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0327 15:18:11.838822 4011 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0327 15:18:18.765765 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:18:18.765794 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:18:18.765803 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:18:18.794876 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 15:18:28.696306 4011 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-q2vlx] creating logical port openshift-pipelines_tekton-triggers-webhook-55d94ff6c6-q2vlx for pod on switch ip-10-0-2-39.ec2.internal I0327 15:18:28.696422 4011 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-q2vlx I0327 15:18:28.714803 4011 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-q2vlx] pod update took 18.398246ms I0327 15:18:28.714822 4011 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-q2vlx] addLogicalPort annotation time took 18.425041ms I0327 15:18:28.716172 4011 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-q2vlx] addLogicalPort took 19.88506ms, libovsdb time 1.015492ms I0327 15:18:29.149643 4011 cni.go:403] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-q2vlx 8eb6a1d018d527c3f37f86c8ff6c47c374ee18f9a52a319b80500d5a012b92b2 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-q2vlx 8eb6a1d018d527c3f37f86c8ff6c47c374ee18f9a52a319b80500d5a012b92b2 network default NAD default] I0327 15:18:29.158230 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-55d94ff6c6-q2vlx, hostIfaceName: 8eb6a1d018d527c, network: default, NAD default, SandboxID: "8eb6a1d018d527c3f37f86c8ff6c47c374ee18f9a52a319b80500d5a012b92b2", PCI device ID: , UID: "55fd95ae-a5c7-4436-a554-e4708e0a10ba", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0327 15:18:29.202055 4011 cni.go:424] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-q2vlx 8eb6a1d018d527c3f37f86c8ff6c47c374ee18f9a52a319b80500d5a012b92b2 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-q2vlx 8eb6a1d018d527c3f37f86c8ff6c47c374ee18f9a52a319b80500d5a012b92b2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:8d:11:4f:fc:7f\",\"name\":\"8eb6a1d018d527c\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/225c738a-2684-4a1f-b2e7-576aa27c17ff\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:18:32.225277 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 15:18:35.946087 4011 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0327 15:18:36.028878 4011 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:36.029423 4011 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:36.049711 4011 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:18:36.049730 4011 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:18:36.056159 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0327 15:18:36.056184 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0327 15:18:36.936943 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=43 W0327 15:18:42.948885 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:18:42.948991 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:42.949008 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0327 15:18:50.183164 4011 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 15:18:53.254732 4011 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:18:53.254862 4011 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0327 15:18:53.267998 4011 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 13.159533ms I0327 15:18:53.268018 4011 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 13.185249ms I0327 15:18:53.269222 4011 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 14.508509ms, libovsdb time 845.48µs I0327 15:18:57.211815 4011 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 2dd8e4438bb9284e8cc36b15a236ad3482b91d70f024854e2476274ec6e834c4 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 2dd8e4438bb9284e8cc36b15a236ad3482b91d70f024854e2476274ec6e834c4 network default NAD default] I0327 15:18:57.219915 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 2dd8e4438bb9284, network: default, NAD default, SandboxID: "2dd8e4438bb9284e8cc36b15a236ad3482b91d70f024854e2476274ec6e834c4", PCI device ID: , UID: "99e3b4dd-009e-4525-aa7b-80c7169dcd47", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0327 15:18:57.263739 4011 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 2dd8e4438bb9284e8cc36b15a236ad3482b91d70f024854e2476274ec6e834c4 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 2dd8e4438bb9284e8cc36b15a236ad3482b91d70f024854e2476274ec6e834c4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:80:82:4c:90:5b\",\"name\":\"2dd8e4438bb9284\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2b137293-7735-41ec-90bf-5f3f9ccf7b75\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:19:03.086384 4011 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 15:19:03.329020 4011 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:03.329656 4011 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:04.076867 4011 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:04.076888 4011 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:04.082346 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 15:19:04.082374 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 15:19:04.529163 4011 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-snmn9] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-55cbb85cb8-snmn9 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:19:04.529242 4011 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-snmn9 I0327 15:19:04.543063 4011 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-snmn9] pod update took 13.841238ms I0327 15:19:04.543095 4011 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-snmn9] addLogicalPort annotation time took 13.875358ms I0327 15:19:04.544557 4011 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-snmn9] addLogicalPort took 15.411642ms, libovsdb time 914.719µs I0327 15:19:04.630453 4011 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-jdnvs] creating logical port openshift-pipelines_tekton-results-watcher-6f946f97cf-jdnvs for pod on switch ip-10-0-2-39.ec2.internal I0327 15:19:04.630585 4011 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-6f946f97cf-jdnvs I0327 15:19:04.645415 4011 pod.go:62] [openshift-pipelines/tekton-results-watcher-6f946f97cf-jdnvs] pod update took 14.848777ms I0327 15:19:04.645441 4011 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-jdnvs] addLogicalPort annotation time took 14.898266ms I0327 15:19:04.646712 4011 pods.go:271] [openshift-pipelines/tekton-results-watcher-6f946f97cf-jdnvs] addLogicalPort took 16.275663ms, libovsdb time 886.278µs I0327 15:19:04.894672 4011 cni.go:403] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-snmn9 a74d3b00d22e2bc5789b164dd94f70bd2c57239050a615573846a398d2a5cbe0 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-snmn9 a74d3b00d22e2bc5789b164dd94f70bd2c57239050a615573846a398d2a5cbe0 network default NAD default] I0327 15:19:04.903489 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-55cbb85cb8-snmn9, hostIfaceName: a74d3b00d22e2bc, network: default, NAD default, SandboxID: "a74d3b00d22e2bc5789b164dd94f70bd2c57239050a615573846a398d2a5cbe0", PCI device ID: , UID: "2b275427-307b-4543-8e76-8bc374ef25be", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0327 15:19:04.947336 4011 cni.go:424] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-snmn9 a74d3b00d22e2bc5789b164dd94f70bd2c57239050a615573846a398d2a5cbe0 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-snmn9 a74d3b00d22e2bc5789b164dd94f70bd2c57239050a615573846a398d2a5cbe0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:93:d5:5e:42:c6\",\"name\":\"a74d3b00d22e2bc\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e98d7b28-0b8c-47c7-892b-3c5936b33757\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:19:04.973475 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0327 15:19:04.993871 4011 cni.go:403] [openshift-pipelines/tekton-results-watcher-6f946f97cf-jdnvs d0388f87cbc6ba5f8de1f0f5f6312b6df47a0e7bb205fceb1b42b3eae5755ba7 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-jdnvs d0388f87cbc6ba5f8de1f0f5f6312b6df47a0e7bb205fceb1b42b3eae5755ba7 network default NAD default] I0327 15:19:05.002447 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-6f946f97cf-jdnvs, hostIfaceName: d0388f87cbc6ba5, network: default, NAD default, SandboxID: "d0388f87cbc6ba5f8de1f0f5f6312b6df47a0e7bb205fceb1b42b3eae5755ba7", PCI device ID: , UID: "d0803edd-dac4-4d8b-ad76-fa43ddf90db8", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0327 15:19:05.045621 4011 cni.go:424] [openshift-pipelines/tekton-results-watcher-6f946f97cf-jdnvs d0388f87cbc6ba5f8de1f0f5f6312b6df47a0e7bb205fceb1b42b3eae5755ba7 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-jdnvs d0388f87cbc6ba5f8de1f0f5f6312b6df47a0e7bb205fceb1b42b3eae5755ba7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:d4:e0:ad:68:60\",\"name\":\"d0388f87cbc6ba5\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7fb2e8cf-2e78-41f1-9336-d03618a74505\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:19:06.108525 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:06.108573 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:06.109142 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:19:09.402617 4011 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:19:09.402635 4011 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:19:09.402641 4011 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0327 15:19:14.304619 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0327 15:19:15.665515 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0327 15:19:26.009702 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=96 I0327 15:19:34.428934 4011 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2cfqc] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-2cfqc for pod on switch ip-10-0-2-39.ec2.internal I0327 15:19:34.429036 4011 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2cfqc I0327 15:19:34.443405 4011 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2cfqc] pod update took 14.39484ms I0327 15:19:34.443426 4011 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2cfqc] addLogicalPort annotation time took 14.425437ms I0327 15:19:34.444746 4011 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2cfqc] addLogicalPort took 15.826842ms, libovsdb time 841.332µs I0327 15:19:34.619706 4011 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-4w8ww] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-4w8ww for pod on switch ip-10-0-2-39.ec2.internal I0327 15:19:34.619780 4011 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-4w8ww I0327 15:19:34.636528 4011 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-4w8ww] pod update took 16.765186ms I0327 15:19:34.636569 4011 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-4w8ww] addLogicalPort annotation time took 16.80967ms I0327 15:19:34.637886 4011 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-4w8ww] addLogicalPort took 18.19034ms, libovsdb time 940.727µs I0327 15:19:34.792151 4011 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2cfqc 4157d652bd03ac6926a9a2b4cf00f48e07456e9a4bb39ecd1cb1fa8d8c8b1937 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2cfqc 4157d652bd03ac6926a9a2b4cf00f48e07456e9a4bb39ecd1cb1fa8d8c8b1937 network default NAD default] I0327 15:19:34.800632 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-2cfqc, hostIfaceName: 4157d652bd03ac6, network: default, NAD default, SandboxID: "4157d652bd03ac6926a9a2b4cf00f48e07456e9a4bb39ecd1cb1fa8d8c8b1937", PCI device ID: , UID: "98925e53-9341-49cf-b272-cf06e62a3fc7", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0327 15:19:34.846567 4011 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2cfqc 4157d652bd03ac6926a9a2b4cf00f48e07456e9a4bb39ecd1cb1fa8d8c8b1937 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-2cfqc 4157d652bd03ac6926a9a2b4cf00f48e07456e9a4bb39ecd1cb1fa8d8c8b1937 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:f0:31:1e:a2:e2\",\"name\":\"4157d652bd03ac6\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0ff8b84-20db-457b-8435-b5ceffee86af\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:19:34.989856 4011 cni.go:403] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-4w8ww 5d4e80739fa2a940123ee22714586ccbabc0f088a9ee88453fd36e6db1cc146e network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-4w8ww 5d4e80739fa2a940123ee22714586ccbabc0f088a9ee88453fd36e6db1cc146e network default NAD default] I0327 15:19:34.998405 4011 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-4w8ww, hostIfaceName: 5d4e80739fa2a94, network: default, NAD default, SandboxID: "5d4e80739fa2a940123ee22714586ccbabc0f088a9ee88453fd36e6db1cc146e", PCI device ID: , UID: "9a658eaa-7904-40c7-a3e0-1007133eadf1", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0327 15:19:35.042229 4011 cni.go:424] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-4w8ww 5d4e80739fa2a940123ee22714586ccbabc0f088a9ee88453fd36e6db1cc146e network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-4w8ww 5d4e80739fa2a940123ee22714586ccbabc0f088a9ee88453fd36e6db1cc146e network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:66:f5:0c:8a:a2\",\"name\":\"5d4e80739fa2a94\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7aafad10-cdf6-4a39-82fa-e98e29df83ca\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:19:37.535907 4011 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0327 15:19:38.494488 4011 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:38.495042 4011 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:44.490158 4011 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-bflzg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:44.490775 4011 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-bflzg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:45.514651 4011 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-bflzg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:45.514674 4011 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-bflzg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:45.870116 4011 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:45.870679 4011 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:46.361791 4011 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:46.361820 4011 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:46.367840 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 15:19:46.367872 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0327 15:19:48.385124 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:19:48.388294 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:48.388323 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0327 15:20:01.646332 4011 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:01.646974 4011 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:01.655255 4011 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:01.655274 4011 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:01.661740 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq I0327 15:20:01.661761 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq I0327 15:20:05.646420 4011 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-pjn4l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:05.646987 4011 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-pjn4l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:06.531579 4011 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-pjn4l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:06.531611 4011 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-pjn4l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:13.688790 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=107 I0327 15:20:38.610217 4011 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:38.610247 4011 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:38.615913 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 15:20:38.615931 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 15:20:46.879274 4011 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:46.879881 4011 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:46.901386 4011 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:46.901410 4011 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:46.906785 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 15:20:46.906813 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 15:20:49.539633 4011 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-744d6f496c-fsfpx] creating logical port tsf-tas_trillian-logserver-744d6f496c-fsfpx for pod on switch ip-10-0-2-39.ec2.internal I0327 15:20:49.539749 4011 kube.go:256] Updating pod tsf-tas/trillian-logserver-744d6f496c-fsfpx I0327 15:20:49.557732 4011 pod.go:62] [tsf-tas/trillian-logserver-744d6f496c-fsfpx] pod update took 18.004251ms I0327 15:20:49.557760 4011 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logserver-744d6f496c-fsfpx] addLogicalPort annotation time took 18.035981ms I0327 15:20:49.559177 4011 pods.go:271] [tsf-tas/trillian-logserver-744d6f496c-fsfpx] addLogicalPort took 19.559121ms, libovsdb time 913.549µs I0327 15:20:50.502883 4011 cni.go:403] [tsf-tas/trillian-logserver-744d6f496c-fsfpx fcdc343e67926272e8773bac56926d845db6f74a5e28af949f1a130cf7befcaf network default NAD default] ADD starting CNI request [tsf-tas/trillian-logserver-744d6f496c-fsfpx fcdc343e67926272e8773bac56926d845db6f74a5e28af949f1a130cf7befcaf network default NAD default] I0327 15:20:50.511144 4011 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-744d6f496c-fsfpx, hostIfaceName: fcdc343e6792627, network: default, NAD default, SandboxID: "fcdc343e67926272e8773bac56926d845db6f74a5e28af949f1a130cf7befcaf", PCI device ID: , UID: "84cfc6c7-17d1-4e23-b8b2-0ed41326a2f2", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0327 15:20:50.557433 4011 cni.go:424] [tsf-tas/trillian-logserver-744d6f496c-fsfpx fcdc343e67926272e8773bac56926d845db6f74a5e28af949f1a130cf7befcaf network default NAD default] ADD finished CNI request [tsf-tas/trillian-logserver-744d6f496c-fsfpx fcdc343e67926272e8773bac56926d845db6f74a5e28af949f1a130cf7befcaf network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:bc:22:35:2f:48\",\"name\":\"fcdc343e6792627\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d32949b0-1fc5-461b-a68f-c98b97ca83cf\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:20:52.389732 4011 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-5fcb46748d-njq87] creating logical port tsf-tas_trillian-db-5fcb46748d-njq87 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:20:52.389838 4011 kube.go:256] Updating pod tsf-tas/trillian-db-5fcb46748d-njq87 I0327 15:20:52.402773 4011 pod.go:62] [tsf-tas/trillian-db-5fcb46748d-njq87] pod update took 12.95804ms I0327 15:20:52.402798 4011 base_network_controller_pods.go:937] [default/tsf-tas/trillian-db-5fcb46748d-njq87] addLogicalPort annotation time took 12.988726ms I0327 15:20:52.404325 4011 pods.go:271] [tsf-tas/trillian-db-5fcb46748d-njq87] addLogicalPort took 14.613147ms, libovsdb time 1.021388ms W0327 15:20:56.649817 4011 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0327 15:20:56.919125 4011 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0327 15:20:58.867628 4011 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0327 15:20:58.867644 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0327 15:20:59.952849 4011 cni.go:403] [tsf-tas/trillian-db-5fcb46748d-njq87 a44e4c3ce54d34a9ee6849b6db876d4b2f2fa16c748f8c0b2439a22a172a9a67 network default NAD default] ADD starting CNI request [tsf-tas/trillian-db-5fcb46748d-njq87 a44e4c3ce54d34a9ee6849b6db876d4b2f2fa16c748f8c0b2439a22a172a9a67 network default NAD default] I0327 15:20:59.961020 4011 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-5fcb46748d-njq87, hostIfaceName: a44e4c3ce54d34a, network: default, NAD default, SandboxID: "a44e4c3ce54d34a9ee6849b6db876d4b2f2fa16c748f8c0b2439a22a172a9a67", PCI device ID: , UID: "9a83b1b1-5b11-4421-ab09-62bb006a40c5", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0327 15:21:00.007376 4011 cni.go:424] [tsf-tas/trillian-db-5fcb46748d-njq87 a44e4c3ce54d34a9ee6849b6db876d4b2f2fa16c748f8c0b2439a22a172a9a67 network default NAD default] ADD finished CNI request [tsf-tas/trillian-db-5fcb46748d-njq87 a44e4c3ce54d34a9ee6849b6db876d4b2f2fa16c748f8c0b2439a22a172a9a67 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:ee:d0:5b:60:84\",\"name\":\"a44e4c3ce54d34a\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2a7b1246-537b-42d5-ad96-3ab98a85e304\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:21:02.006669 4011 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ns4gd-lf892 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:02.007263 4011 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ns4gd-lf892 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:02.883886 4011 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ns4gd-lf892 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:02.883911 4011 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ns4gd-lf892 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:21:03.689993 4011 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0327 15:21:08.793508 4011 namespace.go:142] [openshift-pipelines] updating namespace I0327 15:21:13.735209 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 W0327 15:21:14.733201 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql I0327 15:21:15.826880 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=63 I0327 15:21:16.817776 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 W0327 15:21:30.784134 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0327 15:21:42.244471 4011 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:42.245043 4011 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:42.262837 4011 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:42.262854 4011 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:42.265690 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0327 15:21:42.265713 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0327 15:21:44.296623 4011 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-c4mtb-wb5d4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:44.297179 4011 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-c4mtb-wb5d4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:45.164599 4011 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-c4mtb-wb5d4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:45.164627 4011 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-c4mtb-wb5d4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:45.287110 4011 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-4hn7z-bj6gp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:45.287730 4011 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-4hn7z-bj6gp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:46.179502 4011 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-4hn7z-bj6gp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:46.179523 4011 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-4hn7z-bj6gp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:21:48.172105 4011 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0327 15:21:51.738497 4011 obj_retry.go:464] Detected object tsf/patch-tekton-config-rv5qd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:51.739120 4011 obj_retry.go:464] Detected object tsf/patch-tekton-config-rv5qd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:52.757314 4011 obj_retry.go:458] Detected object tsf/patch-tekton-config-rv5qd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:52.757334 4011 obj_retry.go:458] Detected object tsf/patch-tekton-config-rv5qd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:21:53.188072 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0327 15:21:53.205102 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0327 15:21:58.218591 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0327 15:21:58.356416 4011 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-cchnn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:58.356961 4011 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-cchnn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:59.378903 4011 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-cchnn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:59.378927 4011 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-cchnn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:59.415552 4011 obj_retry.go:458] Detected object tsf/patch-tekton-config-rv5qd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:59.415579 4011 obj_retry.go:458] Detected object tsf/patch-tekton-config-rv5qd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:59.425810 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-rv5qd I0327 15:21:59.425828 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-rv5qd I0327 15:22:09.420166 4011 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:09.420774 4011 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:09.436912 4011 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:09.436931 4011 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:09.443922 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0327 15:22:09.443936 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0327 15:22:14.467094 4011 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:14.467764 4011 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:14.486337 4011 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:14.486354 4011 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:14.493488 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0327 15:22:14.493508 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0327 15:22:21.460896 4011 obj_retry.go:464] Detected object tsf-tpa/create-db-4v6df of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:21.461447 4011 obj_retry.go:464] Detected object tsf-tpa/create-db-4v6df of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:22.484914 4011 obj_retry.go:458] Detected object tsf-tpa/create-db-4v6df of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:22.484935 4011 obj_retry.go:458] Detected object tsf-tpa/create-db-4v6df of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:24.494117 4011 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pq86c-dt6kd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:24.494685 4011 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pq86c-dt6kd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.380145 4011 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pq86c-dt6kd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.380174 4011 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pq86c-dt6kd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.515270 4011 obj_retry.go:464] Detected object tsf-tpa/migrate-db-h7chx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.515847 4011 obj_retry.go:464] Detected object tsf-tpa/migrate-db-h7chx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.609694 4011 obj_retry.go:458] Detected object tsf-tpa/migrate-db-h7chx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.609713 4011 obj_retry.go:458] Detected object tsf-tpa/migrate-db-h7chx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.790149 4011 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-9h4cb] creating logical port tsf-tpa_create-importers-9h4cb for pod on switch ip-10-0-2-39.ec2.internal I0327 15:22:25.790245 4011 kube.go:256] Updating pod tsf-tpa/create-importers-9h4cb I0327 15:22:25.802198 4011 pod.go:62] [tsf-tpa/create-importers-9h4cb] pod update took 11.986333ms I0327 15:22:25.802218 4011 base_network_controller_pods.go:937] [default/tsf-tpa/create-importers-9h4cb] addLogicalPort annotation time took 12.01324ms I0327 15:22:25.803422 4011 pods.go:271] [tsf-tpa/create-importers-9h4cb] addLogicalPort took 13.289059ms, libovsdb time 798.421µs I0327 15:22:26.147327 4011 cni.go:403] [tsf-tpa/create-importers-9h4cb d1a6a65a6e22e7c0de3e2f60f547d364f1ae6781972134cab926fa6aa6104974 network default NAD default] ADD starting CNI request [tsf-tpa/create-importers-9h4cb d1a6a65a6e22e7c0de3e2f60f547d364f1ae6781972134cab926fa6aa6104974 network default NAD default] I0327 15:22:26.155597 4011 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-9h4cb, hostIfaceName: d1a6a65a6e22e7c, network: default, NAD default, SandboxID: "d1a6a65a6e22e7c0de3e2f60f547d364f1ae6781972134cab926fa6aa6104974", PCI device ID: , UID: "223eec5b-cc3f-4100-b13e-02d02f38c417", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0327 15:22:26.201139 4011 cni.go:424] [tsf-tpa/create-importers-9h4cb d1a6a65a6e22e7c0de3e2f60f547d364f1ae6781972134cab926fa6aa6104974 network default NAD default] ADD finished CNI request [tsf-tpa/create-importers-9h4cb d1a6a65a6e22e7c0de3e2f60f547d364f1ae6781972134cab926fa6aa6104974 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:76:e7:a7:cf:b0\",\"name\":\"d1a6a65a6e22e7c\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/764d61c3-c183-48eb-be7a-4d71c651a669\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:22:28.201769 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=541 I0327 15:22:31.165314 4011 cni.go:403] [tsf-tpa/create-importers-9h4cb d1a6a65a6e22e7c0de3e2f60f547d364f1ae6781972134cab926fa6aa6104974 network default NAD default] DEL starting CNI request [tsf-tpa/create-importers-9h4cb d1a6a65a6e22e7c0de3e2f60f547d364f1ae6781972134cab926fa6aa6104974 network default NAD default] I0327 15:22:31.241735 4011 cni.go:424] [tsf-tpa/create-importers-9h4cb d1a6a65a6e22e7c0de3e2f60f547d364f1ae6781972134cab926fa6aa6104974 network default NAD default] DEL finished CNI request [tsf-tpa/create-importers-9h4cb d1a6a65a6e22e7c0de3e2f60f547d364f1ae6781972134cab926fa6aa6104974 network default NAD default], result "{}", err I0327 15:22:31.292481 4011 obj_retry.go:464] Detected object tsf-tpa/create-importers-9h4cb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:31.292559 4011 pods.go:176] Deleting pod: tsf-tpa/create-importers-9h4cb I0327 15:22:31.293835 4011 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-9h4cb, ips: 10.128.0.43 I0327 15:22:31.293861 4011 obj_retry.go:464] Detected object tsf-tpa/create-importers-9h4cb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:32.317680 4011 obj_retry.go:458] Detected object tsf-tpa/create-importers-9h4cb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:32.317704 4011 obj_retry.go:458] Detected object tsf-tpa/create-importers-9h4cb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:22:34.417799 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0327 15:22:34.443376 4011 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0327 15:23:02.779395 4011 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:02.779970 4011 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:02.817142 4011 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:02.817164 4011 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:02.823812 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0327 15:23:02.823834 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0327 15:23:03.624598 4011 namespace.go:98] [image-controller] adding namespace I0327 15:23:03.625432 4011 namespace.go:98] [integration-service] adding namespace I0327 15:23:03.625477 4011 namespace.go:98] [build-service] adding namespace I0327 15:23:03.625501 4011 namespace.go:102] [image-controller] adding namespace took 881.439µs I0327 15:23:03.626214 4011 namespace.go:102] [build-service] adding namespace took 703.562µs I0327 15:23:03.626856 4011 namespace.go:102] [integration-service] adding namespace took 1.402608ms I0327 15:23:03.679222 4011 namespace.go:142] [image-controller] updating namespace I0327 15:23:03.711833 4011 namespace.go:142] [image-controller] updating namespace I0327 15:23:03.726718 4011 namespace.go:142] [integration-service] updating namespace I0327 15:23:03.762103 4011 namespace.go:142] [build-service] updating namespace I0327 15:23:03.770785 4011 namespace.go:142] [integration-service] updating namespace I0327 15:23:03.794875 4011 namespace.go:142] [build-service] updating namespace I0327 15:23:03.891845 4011 namespace.go:142] [build-service] updating namespace I0327 15:23:03.931814 4011 namespace.go:142] [integration-service] updating namespace I0327 15:23:03.989054 4011 namespace.go:98] [release-service] adding namespace I0327 15:23:03.989826 4011 namespace.go:102] [release-service] adding namespace took 756.058µs I0327 15:23:04.088781 4011 namespace.go:142] [release-service] updating namespace I0327 15:23:04.088840 4011 namespace.go:142] [konflux-ui] updating namespace I0327 15:23:04.137920 4011 namespace.go:142] [release-service] updating namespace I0327 15:23:04.228223 4011 namespace.go:98] [konflux-info] adding namespace I0327 15:23:04.228979 4011 namespace.go:102] [konflux-info] adding namespace took 739.91µs I0327 15:23:04.334154 4011 namespace.go:98] [namespace-lister] adding namespace I0327 15:23:04.334265 4011 namespace.go:142] [konflux-info] updating namespace I0327 15:23:04.334948 4011 namespace.go:102] [namespace-lister] adding namespace took 779.805µs I0327 15:23:04.383818 4011 namespace.go:98] [enterprise-contract-service] adding namespace I0327 15:23:04.384584 4011 namespace.go:102] [enterprise-contract-service] adding namespace took 750.969µs I0327 15:23:04.420808 4011 namespace.go:142] [image-controller] updating namespace I0327 15:23:04.524494 4011 namespace.go:142] [konflux-info] updating namespace I0327 15:23:04.524576 4011 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:04.641869 4011 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:04.717912 4011 namespace.go:98] [default-tenant] adding namespace I0327 15:23:04.717979 4011 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:04.718844 4011 namespace.go:102] [default-tenant] adding namespace took 911.874µs I0327 15:23:04.806888 4011 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:04.827701 4011 namespace.go:98] [segment-bridge] adding namespace I0327 15:23:04.828412 4011 namespace.go:102] [segment-bridge] adding namespace took 695.775µs I0327 15:23:04.877891 4011 namespace.go:142] [default-tenant] updating namespace I0327 15:23:04.961119 4011 namespace.go:142] [default-tenant] updating namespace I0327 15:23:04.969044 4011 namespace.go:142] [segment-bridge] updating namespace I0327 15:23:05.032018 4011 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0327 15:23:05.032578 4011 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0327 15:23:05.032596 4011 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0327 15:23:05.035151 4011 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0327 15:23:05.035848 4011 namespace.go:142] [segment-bridge] updating namespace I0327 15:23:05.080842 4011 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0327 15:23:05.080863 4011 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0327 15:23:05.081525 4011 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0327 15:23:05.141162 4011 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-67655c4674-z559l] creating logical port namespace-lister_namespace-lister-67655c4674-z559l for pod on switch ip-10-0-2-39.ec2.internal I0327 15:23:05.141241 4011 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-z559l I0327 15:23:05.174280 4011 pod.go:62] [namespace-lister/namespace-lister-67655c4674-z559l] pod update took 33.061621ms I0327 15:23:05.174299 4011 base_network_controller_pods.go:937] [default/namespace-lister/namespace-lister-67655c4674-z559l] addLogicalPort annotation time took 33.087463ms I0327 15:23:05.175475 4011 pods.go:271] [namespace-lister/namespace-lister-67655c4674-z559l] addLogicalPort took 34.326113ms, libovsdb time 788.206µs I0327 15:23:05.315173 4011 namespace.go:142] [build-service] updating namespace I0327 15:23:05.530888 4011 base_network_controller_pods.go:487] [default/konflux-ui/dex-79786448fd-nztkx] creating logical port konflux-ui_dex-79786448fd-nztkx for pod on switch ip-10-0-2-39.ec2.internal I0327 15:23:05.530975 4011 kube.go:256] Updating pod konflux-ui/dex-79786448fd-nztkx I0327 15:23:05.579750 4011 pod.go:62] [konflux-ui/dex-79786448fd-nztkx] pod update took 48.798463ms I0327 15:23:05.579774 4011 base_network_controller_pods.go:937] [default/konflux-ui/dex-79786448fd-nztkx] addLogicalPort annotation time took 48.825312ms I0327 15:23:05.580979 4011 pods.go:271] [konflux-ui/dex-79786448fd-nztkx] addLogicalPort took 50.105188ms, libovsdb time 794.894µs I0327 15:23:05.688095 4011 base_network_controller_pods.go:487] [default/konflux-ui/proxy-658b44b84-npv7x] creating logical port konflux-ui_proxy-658b44b84-npv7x for pod on switch ip-10-0-2-39.ec2.internal I0327 15:23:05.688222 4011 kube.go:256] Updating pod konflux-ui/proxy-658b44b84-npv7x I0327 15:23:05.720712 4011 namespace.go:142] [image-controller] updating namespace I0327 15:23:05.769298 4011 pod.go:62] [konflux-ui/proxy-658b44b84-npv7x] pod update took 81.108009ms I0327 15:23:05.769324 4011 base_network_controller_pods.go:937] [default/konflux-ui/proxy-658b44b84-npv7x] addLogicalPort annotation time took 81.139644ms I0327 15:23:05.770677 4011 pods.go:271] [konflux-ui/proxy-658b44b84-npv7x] addLogicalPort took 82.59385ms, libovsdb time 655.995µs I0327 15:23:05.943313 4011 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-5f8cd64d55-nwslp] creating logical port image-controller_image-controller-controller-manager-5f8cd64d55-nwslp for pod on switch ip-10-0-2-39.ec2.internal I0327 15:23:05.943472 4011 kube.go:256] Updating pod image-controller/image-controller-controller-manager-5f8cd64d55-nwslp I0327 15:23:05.995493 4011 pod.go:62] [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp] pod update took 52.074652ms I0327 15:23:05.995522 4011 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-5f8cd64d55-nwslp] addLogicalPort annotation time took 52.111501ms I0327 15:23:05.996937 4011 pods.go:271] [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp] addLogicalPort took 53.638916ms, libovsdb time 962.258µs I0327 15:23:06.222274 4011 namespace.go:142] [integration-service] updating namespace I0327 15:23:06.426975 4011 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7df75dd4df-d4sm2] creating logical port build-service_build-service-controller-manager-7df75dd4df-d4sm2 for pod on switch ip-10-0-2-39.ec2.internal I0327 15:23:06.427104 4011 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-d4sm2 I0327 15:23:06.455908 4011 pod.go:62] [build-service/build-service-controller-manager-7df75dd4df-d4sm2] pod update took 28.831118ms I0327 15:23:06.455935 4011 base_network_controller_pods.go:937] [default/build-service/build-service-controller-manager-7df75dd4df-d4sm2] addLogicalPort annotation time took 28.863838ms I0327 15:23:06.457284 4011 pods.go:271] [build-service/build-service-controller-manager-7df75dd4df-d4sm2] addLogicalPort took 30.328842ms, libovsdb time 852.904µs I0327 15:23:06.610849 4011 namespace.go:142] [konflux-info] updating namespace I0327 15:23:06.792195 4011 cni.go:403] [build-service/build-service-controller-manager-7df75dd4df-d4sm2 3f6d8bc73523780c96f3d6aa252d7750705fa44db3ef5722ec5583f96c28f4c8 network default NAD default] ADD starting CNI request [build-service/build-service-controller-manager-7df75dd4df-d4sm2 3f6d8bc73523780c96f3d6aa252d7750705fa44db3ef5722ec5583f96c28f4c8 network default NAD default] I0327 15:23:06.800726 4011 helper_linux.go:450] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7df75dd4df-d4sm2, hostIfaceName: 3f6d8bc73523780, network: default, NAD default, SandboxID: "3f6d8bc73523780c96f3d6aa252d7750705fa44db3ef5722ec5583f96c28f4c8", PCI device ID: , UID: "4da82d6f-cb45-45b5-b8c2-e513d6aab074", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0327 15:23:06.848521 4011 cni.go:424] [build-service/build-service-controller-manager-7df75dd4df-d4sm2 3f6d8bc73523780c96f3d6aa252d7750705fa44db3ef5722ec5583f96c28f4c8 network default NAD default] ADD finished CNI request [build-service/build-service-controller-manager-7df75dd4df-d4sm2 3f6d8bc73523780c96f3d6aa252d7750705fa44db3ef5722ec5583f96c28f4c8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:3c:98:34:32:c4\",\"name\":\"3f6d8bc73523780\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/446ed204-8128-4757-a365-2f80ee18266a\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:23:06.931021 4011 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:07.157070 4011 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-854f775895-47rlg] creating logical port release-service_release-service-controller-manager-854f775895-47rlg for pod on switch ip-10-0-2-39.ec2.internal I0327 15:23:07.157167 4011 kube.go:256] Updating pod release-service/release-service-controller-manager-854f775895-47rlg I0327 15:23:07.200955 4011 namespace.go:142] [release-service] updating namespace I0327 15:23:07.203742 4011 pod.go:62] [release-service/release-service-controller-manager-854f775895-47rlg] pod update took 46.60018ms I0327 15:23:07.203763 4011 base_network_controller_pods.go:937] [default/release-service/release-service-controller-manager-854f775895-47rlg] addLogicalPort annotation time took 46.624347ms I0327 15:23:07.204942 4011 pods.go:271] [release-service/release-service-controller-manager-854f775895-47rlg] addLogicalPort took 47.886763ms, libovsdb time 779.968µs I0327 15:23:07.344120 4011 namespace.go:142] [build-service] updating namespace I0327 15:23:07.365183 4011 namespace.go:142] [image-controller] updating namespace I0327 15:23:07.402364 4011 namespace.go:142] [integration-service] updating namespace I0327 15:23:07.465165 4011 namespace.go:142] [konflux-info] updating namespace I0327 15:23:07.495993 4011 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:07.512752 4011 namespace.go:142] [release-service] updating namespace I0327 15:23:07.585864 4011 namespace.go:142] [build-service] updating namespace I0327 15:23:07.724039 4011 namespace.go:142] [image-controller] updating namespace I0327 15:23:07.838599 4011 namespace.go:142] [integration-service] updating namespace I0327 15:23:07.852614 4011 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-9rx2b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:07.853271 4011 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-9rx2b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:07.928070 4011 namespace.go:142] [konflux-info] updating namespace I0327 15:23:08.082007 4011 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:08.093044 4011 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp 964a868fdf5c4b9eb9ef877b52dd9a2f96f14042088f9024f871fd099d541b61 network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp 964a868fdf5c4b9eb9ef877b52dd9a2f96f14042088f9024f871fd099d541b61 network default NAD default] I0327 15:23:08.101879 4011 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-5f8cd64d55-nwslp, hostIfaceName: 964a868fdf5c4b9, network: default, NAD default, SandboxID: "964a868fdf5c4b9eb9ef877b52dd9a2f96f14042088f9024f871fd099d541b61", PCI device ID: , UID: "7c75cd5e-502a-40dc-989f-561902ccbb67", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0327 15:23:08.147063 4011 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp 964a868fdf5c4b9eb9ef877b52dd9a2f96f14042088f9024f871fd099d541b61 network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp 964a868fdf5c4b9eb9ef877b52dd9a2f96f14042088f9024f871fd099d541b61 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:82:8b:cf:9f:21\",\"name\":\"964a868fdf5c4b9\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9c44eb71-9494-4a15-93fe-650e403a8886\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:23:08.185009 4011 namespace.go:142] [release-service] updating namespace I0327 15:23:08.852350 4011 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-9rx2b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:08.852373 4011 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-9rx2b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:09.330440 4011 namespace.go:142] [default-tenant] updating namespace I0327 15:23:09.537325 4011 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:09.733038 4011 namespace.go:142] [segment-bridge] updating namespace I0327 15:23:09.853256 4011 namespace.go:142] [default-tenant] updating namespace I0327 15:23:09.868930 4011 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:09.882039 4011 namespace.go:142] [segment-bridge] updating namespace I0327 15:23:09.963436 4011 namespace.go:142] [default-tenant] updating namespace I0327 15:23:10.006574 4011 cni.go:403] [namespace-lister/namespace-lister-67655c4674-z559l 0e989c58784ebd31536f95a6471eefde6e25e166b6609d2deaeac5fb8e2a79d7 network default NAD default] ADD starting CNI request [namespace-lister/namespace-lister-67655c4674-z559l 0e989c58784ebd31536f95a6471eefde6e25e166b6609d2deaeac5fb8e2a79d7 network default NAD default] I0327 15:23:10.016774 4011 helper_linux.go:450] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-67655c4674-z559l, hostIfaceName: 0e989c58784ebd3, network: default, NAD default, SandboxID: "0e989c58784ebd31536f95a6471eefde6e25e166b6609d2deaeac5fb8e2a79d7", PCI device ID: , UID: "65c33eb4-815d-4abf-a71d-866d321a568e", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0327 15:23:10.068481 4011 cni.go:424] [namespace-lister/namespace-lister-67655c4674-z559l 0e989c58784ebd31536f95a6471eefde6e25e166b6609d2deaeac5fb8e2a79d7 network default NAD default] ADD finished CNI request [namespace-lister/namespace-lister-67655c4674-z559l 0e989c58784ebd31536f95a6471eefde6e25e166b6609d2deaeac5fb8e2a79d7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:8a:b4:46:4c:4c\",\"name\":\"0e989c58784ebd3\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f13e9a8b-d4f6-4012-b760-03c81269c5d6\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:23:10.068883 4011 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:10.169033 4011 namespace.go:142] [segment-bridge] updating namespace W0327 15:23:10.375206 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 15:23:12.388796 4011 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 15:23:12.417328 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0327 15:23:13.392270 4011 cni.go:403] [konflux-ui/dex-79786448fd-nztkx 6dcf6ecd3f05419780887550ba683abc4bb474e337b55e327374fbf893a7db45 network default NAD default] ADD starting CNI request [konflux-ui/dex-79786448fd-nztkx 6dcf6ecd3f05419780887550ba683abc4bb474e337b55e327374fbf893a7db45 network default NAD default] W0327 15:23:13.399316 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0327 15:23:13.401218 4011 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: dex-79786448fd-nztkx, hostIfaceName: 6dcf6ecd3f05419, network: default, NAD default, SandboxID: "6dcf6ecd3f05419780887550ba683abc4bb474e337b55e327374fbf893a7db45", PCI device ID: , UID: "1aa0e066-004b-4fc4-8056-55176e991a0d", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0327 15:23:13.455992 4011 cni.go:424] [konflux-ui/dex-79786448fd-nztkx 6dcf6ecd3f05419780887550ba683abc4bb474e337b55e327374fbf893a7db45 network default NAD default] ADD finished CNI request [konflux-ui/dex-79786448fd-nztkx 6dcf6ecd3f05419780887550ba683abc4bb474e337b55e327374fbf893a7db45 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:33:e6:e1:60:3c\",\"name\":\"6dcf6ecd3f05419\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/801cb15a-f17b-44a0-95bd-f2bc9c56933c\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:23:13.549945 4011 cni.go:403] [konflux-ui/proxy-658b44b84-npv7x 380508dba82068097b68cd28adea06c955f596f989adf4c387439918b964d2b9 network default NAD default] ADD starting CNI request [konflux-ui/proxy-658b44b84-npv7x 380508dba82068097b68cd28adea06c955f596f989adf4c387439918b964d2b9 network default NAD default] I0327 15:23:13.558212 4011 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-658b44b84-npv7x, hostIfaceName: 380508dba820680, network: default, NAD default, SandboxID: "380508dba82068097b68cd28adea06c955f596f989adf4c387439918b964d2b9", PCI device ID: , UID: "a15df73f-e6a2-4dd8-9492-4ea28b39593e", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0327 15:23:13.604437 4011 cni.go:424] [konflux-ui/proxy-658b44b84-npv7x 380508dba82068097b68cd28adea06c955f596f989adf4c387439918b964d2b9 network default NAD default] ADD finished CNI request [konflux-ui/proxy-658b44b84-npv7x 380508dba82068097b68cd28adea06c955f596f989adf4c387439918b964d2b9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:8a:c7:2c:66:61\",\"name\":\"380508dba820680\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/439a9e3e-0949-4e91-b3ad-82acef1457d7\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0327 15:23:15.070084 4011 cni.go:403] [release-service/release-service-controller-manager-854f775895-47rlg 52d9b30020ebc5f47cea8ab7feed4af2e9f1a8a80d82bafefa88fb8ee0c4b9a2 network default NAD default] ADD starting CNI request [release-service/release-service-controller-manager-854f775895-47rlg 52d9b30020ebc5f47cea8ab7feed4af2e9f1a8a80d82bafefa88fb8ee0c4b9a2 network default NAD default] I0327 15:23:15.080156 4011 helper_linux.go:450] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-854f775895-47rlg, hostIfaceName: 52d9b30020ebc5f, network: default, NAD default, SandboxID: "52d9b30020ebc5f47cea8ab7feed4af2e9f1a8a80d82bafefa88fb8ee0c4b9a2", PCI device ID: , UID: "42ae6345-a4de-42d8-b98d-95d44f736762", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0327 15:23:15.134134 4011 cni.go:424] [release-service/release-service-controller-manager-854f775895-47rlg 52d9b30020ebc5f47cea8ab7feed4af2e9f1a8a80d82bafefa88fb8ee0c4b9a2 network default NAD default] ADD finished CNI request [release-service/release-service-controller-manager-854f775895-47rlg 52d9b30020ebc5f47cea8ab7feed4af2e9f1a8a80d82bafefa88fb8ee0c4b9a2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:37:d9:14:45:5c\",\"name\":\"52d9b30020ebc5f\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/402d4bec-c7a7-4239-8c77-7cd89007a60d\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0327 15:23:16.746477 4011 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:23:16.749159 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:23:16.764171 4011 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:23:16.764983 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:23:19.443472 4011 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 15:23:19.443488 4011 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 15:23:19.462661 4011 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0327 15:23:20.448968 4011 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0327 15:23:20.453804 4011 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0327 15:23:21.384190 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 15:23:23.399441 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 15:23:23.821782 4011 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 15:23:23.821799 4011 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0327 15:23:28.084717 4011 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-zpmqr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:28.085247 4011 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-zpmqr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:28.829306 4011 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-zpmqr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:28.829331 4011 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-zpmqr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:28.832288 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-zpmqr I0327 15:23:28.832314 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-zpmqr W0327 15:23:31.459567 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0327 15:23:31.462924 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0327 15:23:32.463271 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0327 15:23:55.501250 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 15:23:55.501280 4011 util.go:959] Failed to build node endpoints for node ip-10-0-2-39.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0327 15:23:59.147495 4011 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 15:24:14.237722 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 15:24:32.197404 4011 cni.go:403] [konflux-ui/proxy-658b44b84-npv7x 380508dba82068097b68cd28adea06c955f596f989adf4c387439918b964d2b9 network default NAD default] DEL starting CNI request [konflux-ui/proxy-658b44b84-npv7x 380508dba82068097b68cd28adea06c955f596f989adf4c387439918b964d2b9 network default NAD default] I0327 15:24:32.280570 4011 cni.go:424] [konflux-ui/proxy-658b44b84-npv7x 380508dba82068097b68cd28adea06c955f596f989adf4c387439918b964d2b9 network default NAD default] DEL finished CNI request [konflux-ui/proxy-658b44b84-npv7x 380508dba82068097b68cd28adea06c955f596f989adf4c387439918b964d2b9 network default NAD default], result "{}", err I0327 15:24:32.349760 4011 obj_retry.go:464] Detected object konflux-ui/proxy-658b44b84-npv7x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:32.349798 4011 pods.go:176] Deleting pod: konflux-ui/proxy-658b44b84-npv7x I0327 15:24:32.351098 4011 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-658b44b84-npv7x, ips: 10.128.0.46 I0327 15:24:32.351127 4011 obj_retry.go:464] Detected object konflux-ui/proxy-658b44b84-npv7x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:32.865160 4011 obj_retry.go:458] Detected object konflux-ui/proxy-658b44b84-npv7x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:32.865190 4011 obj_retry.go:458] Detected object konflux-ui/proxy-658b44b84-npv7x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:32.872036 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-658b44b84-npv7x I0327 15:24:32.872072 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-658b44b84-npv7x I0327 15:24:37.939960 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=54 I0327 15:24:40.183478 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0327 15:24:41.416821 4011 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:41.417390 4011 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:41.435567 4011 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:41.435587 4011 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:41.440761 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0327 15:24:41.440782 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0327 15:25:04.283432 4011 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0327 15:25:09.003411 4011 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 a332e5626de406b786b81e5a7076e2a5d329de123ee0ebd20bbe901884efba73 network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 a332e5626de406b786b81e5a7076e2a5d329de123ee0ebd20bbe901884efba73 network default NAD default] I0327 15:25:09.087706 4011 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 a332e5626de406b786b81e5a7076e2a5d329de123ee0ebd20bbe901884efba73 network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 a332e5626de406b786b81e5a7076e2a5d329de123ee0ebd20bbe901884efba73 network default NAD default], result "{}", err I0327 15:25:09.151778 4011 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:25:09.151808 4011 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 I0327 15:25:09.153137 4011 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-mht92, ips: 10.128.0.28 I0327 15:25:09.153166 4011 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:25:10.067590 4011 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:25:10.067610 4011 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:25:10.076777 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 I0327 15:25:10.076804 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 I0327 15:25:23.672502 4011 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp 964a868fdf5c4b9eb9ef877b52dd9a2f96f14042088f9024f871fd099d541b61 network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp 964a868fdf5c4b9eb9ef877b52dd9a2f96f14042088f9024f871fd099d541b61 network default NAD default] I0327 15:25:23.758186 4011 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp 964a868fdf5c4b9eb9ef877b52dd9a2f96f14042088f9024f871fd099d541b61 network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-nwslp 964a868fdf5c4b9eb9ef877b52dd9a2f96f14042088f9024f871fd099d541b61 network default NAD default], result "{}", err I0327 15:25:23.811799 4011 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-nwslp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:25:23.811834 4011 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-5f8cd64d55-nwslp I0327 15:25:23.813184 4011 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-5f8cd64d55-nwslp, ips: 10.128.0.47 I0327 15:25:23.813211 4011 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-nwslp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:25:24.152354 4011 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-nwslp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:25:24.152376 4011 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-nwslp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:25:24.158981 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-nwslp I0327 15:25:24.159002 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-nwslp I0327 15:25:47.793242 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 15:25:57.308652 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0327 15:26:06.950283 4011 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0327 15:26:45.830631 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=27 I0327 15:28:23.669521 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0327 15:28:41.977331 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0327 15:28:56.695793 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=84 I0327 15:29:05.013730 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=70 I0327 15:29:38.821272 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0327 15:30:04.323520 4011 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-qw9p6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:30:04.324109 4011 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-qw9p6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:30:05.195400 4011 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-qw9p6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:30:05.195445 4011 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-qw9p6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:30:21.943436 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=24 I0327 15:30:31.151308 4011 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 15:30:38.240851 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 15:30:46.204839 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=161 I0327 15:30:48.739263 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0327 15:31:26.187092 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0327 15:32:22.046941 4011 obj_retry.go:458] Detected object tsf-tpa/create-db-4v6df of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:22.046994 4011 obj_retry.go:458] Detected object tsf-tpa/create-db-4v6df of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:22.049393 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-4v6df I0327 15:32:22.049414 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-4v6df I0327 15:32:25.026605 4011 obj_retry.go:458] Detected object tsf-tpa/migrate-db-h7chx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:25.026658 4011 obj_retry.go:458] Detected object tsf-tpa/migrate-db-h7chx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:25.030813 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-h7chx I0327 15:32:25.030845 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-h7chx I0327 15:32:32.046368 4011 obj_retry.go:458] Detected object tsf-tpa/create-importers-9h4cb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:32.046397 4011 obj_retry.go:458] Detected object tsf-tpa/create-importers-9h4cb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:32.048732 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-9h4cb I0327 15:32:32.048758 4011 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-9h4cb I0327 15:33:00.287558 4011 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0327 15:33:15.312051 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0327 15:33:39.796736 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 15:34:07.953981 4011 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0327 15:34:07.981162 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0327 15:34:22.673467 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0327 15:34:42.825376 4011 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0327 15:35:51.834066 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=11 I0327 15:36:13.700185 4011 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=9