+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-103.ec2.internal ++ [[ -n ip-10-0-2-103.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-103.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0326 23:14:58.355022058 - disable conntrack on geneve port' I0326 23:14:58.355022058 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0326 23:14:58.371573780 - starting ovnkube-node + echo 'I0326 23:14:58.371573780 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-103.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-103.ec2.internal --init-node ip-10-0-2-103.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-103.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0326 23:14:58.417560 3840 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0326 23:14:58.417644 3840 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-e7c017ecb5.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0326 23:14:58.419596 3840 kube.go:419] Waiting for certificate I0326 23:14:58.419620 3840 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0326 23:14:58.419652 3840 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0326 23:14:58.420299 3840 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:14:58.447851 3840 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:14:58.447867 3840 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:14:58.451021 3840 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:14:58.454318 3840 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-s98dp" I0326 23:14:58.459542 3840 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-s98dp" I0326 23:14:58.459630 3840 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0326 23:14:59.420261 3840 kube.go:426] Certificate found I0326 23:14:59.420848 3840 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0326 23:14:59.421915 3840 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0326 23:14:59.465520 3840 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-27 23:09:58 +0000 UTC" deadline="2026-03-27 20:44:27.30479833 +0000 UTC" I0326 23:14:59.465706 3840 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="21h29m27.839115527s" I0326 23:14:59.490426 3840 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:14:59.506281 3840 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0326 23:14:59.506479 3840 factory.go:531] Starting watch factory I0326 23:14:59.506600 3840 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506619 3840 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506630 3840 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506646 3840 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506641 3840 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506600 3840 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506721 3840 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506605 3840 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506846 3840 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506860 3840 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506882 3840 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.506667 3840 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.507648 3840 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0326 23:14:59.512515 3840 controller_manager.go:368] Starting the ovnkube controller I0326 23:14:59.512527 3840 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-103.ec2.internal I0326 23:14:59.512599 3840 controller_manager.go:393] NBDB zone sync took: 47.714µs I0326 23:14:59.512608 3840 factory.go:531] Starting watch factory I0326 23:14:59.533713 3840 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.534754 3840 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.536536 3840 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.537966 3840 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.538503 3840 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.542814 3840 ovn_northd.go:121] Registering OVN northd metrics I0326 23:14:59.543141 3840 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0326 23:14:59.546473 3840 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0326 23:14:59.546964 3840 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0326 23:14:59.546988 3840 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0326 23:14:59.563030 3840 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0326 23:14:59.611737 3840 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:14:59.611760 3840 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:14:59.611764 3840 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:14:59.611767 3840 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:14:59.611770 3840 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:14:59.611773 3840 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:14:59.611858 3840 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:14:59.611878 3840 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:14:59.611858 3840 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:14:59.611922 3840 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:14:59.615815 3840 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:14:59.615867 3840 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0326 23:14:59.617841 3840 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:14:59.617854 3840 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:14:59.617911 3840 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.617925 3840 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.618222 3840 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0326 23:14:59.618234 3840 factory.go:1890] *v1.Pod informer cache synced successfully I0326 23:14:59.618238 3840 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0326 23:14:59.618241 3840 factory.go:1890] *v1.Namespace informer cache synced successfully I0326 23:14:59.618244 3840 factory.go:1890] *v1.Node informer cache synced successfully I0326 23:14:59.618247 3840 factory.go:1890] *v1.Service informer cache synced successfully I0326 23:14:59.621080 3840 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.623078 3840 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:14:59.623135 3840 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.623146 3840 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.623320 3840 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0326 23:14:59.623335 3840 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0326 23:14:59.626290 3840 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.628124 3840 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:14:59.628197 3840 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.628213 3840 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.628358 3840 factory.go:1890] *v1.EgressIP informer cache synced successfully I0326 23:14:59.631332 3840 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.633336 3840 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:14:59.633389 3840 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.633401 3840 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.633422 3840 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0326 23:14:59.636602 3840 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.638614 3840 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:14:59.638627 3840 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0326 23:14:59.638669 3840 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.638679 3840 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.641813 3840 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.643777 3840 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:14:59.643865 3840 factory.go:1890] *v1.EgressService informer cache synced successfully I0326 23:14:59.643911 3840 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:14:59.643922 3840 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:14:59.647059 3840 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0326 23:14:59.649039 3840 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:14:59.649048 3840 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0326 23:14:59.649104 3840 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.649108 3840 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.649115 3840 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.649121 3840 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.652356 3840 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.652417 3840 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0326 23:14:59.654440 3840 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:14:59.654450 3840 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:14:59.654457 3840 factory.go:636] Watch Factory start up complete, took: 147.995674ms I0326 23:14:59.654520 3840 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0326 23:14:59.654538 3840 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0326 23:14:59.659864 3840 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0326 23:14:59.659873 3840 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0326 23:14:59.659879 3840 factory.go:636] Watch Factory start up complete, took: 147.270708ms I0326 23:14:59.659886 3840 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-103.ec2.internal" zone I0326 23:14:59.698093 3840 default_node_network_controller.go:709] Initializing the default node network controller I0326 23:14:59.748088 3840 udn_isolation.go:101] Starting UDN host isolation manager I0326 23:14:59.754896 3840 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0326 23:14:59.794781 3840 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0326 23:14:59.794906 3840 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0326 23:14:59.794921 3840 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0326 23:14:59.828538 3840 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0326 23:14:59.828623 3840 default_node_network_controller.go:808] Node ip-10-0-2-103.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0326 23:14:59.834324 3840 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.103"] k8s.ovn.org/zone-name:ip-10-0-2-103.ec2.internal] on node ip-10-0-2-103.ec2.internal I0326 23:14:59.856021 3840 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0326 23:14:59.861498 3840 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0326 23:14:59.861671 3840 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0326 23:14:59.861792 3840 gateway_init.go:263] Preparing Gateway I0326 23:14:59.861804 3840 gateway_shared_intf.go:1658] Creating new gateway I0326 23:14:59.910812 3840 controller_manager.go:427] Waiting for node in zone sync took: 250.911536ms I0326 23:14:59.916370 3840 controller_manager.go:608] Switching to transit router for layer2 networks I0326 23:14:59.916386 3840 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-103.ec2.internal I0326 23:14:59.934969 3840 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0326 23:14:59.937119 3840 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:14:59.940395 3840 controller_manager.go:288] SCTP support detected in OVN I0326 23:14:59.940961 3840 services_controller.go:69] Creating services controller for network=default I0326 23:14:59.941062 3840 udn_enabled_svc.go:80] Waiting for service informer to sync I0326 23:14:59.941170 3840 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0326 23:14:59.941206 3840 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0326 23:14:59.941218 3840 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0326 23:14:59.941226 3840 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:14:59.941380 3840 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0326 23:14:59.941437 3840 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0326 23:14:59.941458 3840 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 46.589µs I0326 23:14:59.941463 3840 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 18.479µs I0326 23:14:59.941483 3840 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 45.199µs I0326 23:14:59.941490 3840 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0326 23:14:59.941497 3840 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0326 23:14:59.941507 3840 default_network_controller.go:321] Starting the default network controller I0326 23:14:59.941549 3840 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 16.462µs I0326 23:14:59.941573 3840 udn_enabled_svc.go:95] Performing full resync I0326 23:14:59.941897 3840 udn_enabled_svc.go:99] Waiting for handler to sync I0326 23:14:59.941911 3840 udn_enabled_svc.go:104] Starting worker I0326 23:14:59.943649 3840 iptables.go:108] Creating table: filter chain: OUTPUT I0326 23:14:59.950018 3840 gateway_shared_intf.go:1787] Gateway Creation Complete I0326 23:14:59.950035 3840 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-103.ec2.internal","mac-address":"0e:ca:83:9e:a3:07","ip-addresses":["10.0.2.103/24"],"ip-address":"10.0.2.103/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:1162c8fa-9a2c-49fa-b71a-82e1dc57bad2 k8s.ovn.org/node-encap-ips:["10.0.2.103"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.103/24"} k8s.ovn.org/zone-name:ip-10-0-2-103.ec2.internal] on node ip-10-0-2-103.ec2.internal I0326 23:14:59.950263 3840 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0326 23:14:59.950320 3840 repair.go:33] Syncing exgw routes took 44.733µs I0326 23:14:59.950338 3840 default_network_controller.go:407] Starting all the Watchers... I0326 23:14:59.950474 3840 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0326 23:14:59.950482 3840 namespace.go:98] [openshift-logging] adding namespace I0326 23:14:59.950490 3840 namespace.go:98] [openshift-dns-operator] adding namespace I0326 23:14:59.950494 3840 namespace.go:98] [openshift-ingress] adding namespace I0326 23:14:59.950501 3840 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0326 23:14:59.950509 3840 namespace.go:98] [openshift-backplane-cse] adding namespace I0326 23:14:59.950518 3840 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0326 23:14:59.950510 3840 namespace.go:98] [openshift] adding namespace I0326 23:14:59.950518 3840 namespace.go:98] [open-cluster-management-2p9mp5cv8t0e1bpnh305hr8ld9shop2c] adding namespace I0326 23:14:59.950524 3840 namespace.go:98] [openshift-backplane] adding namespace I0326 23:14:59.950525 3840 namespace.go:98] [openshift-machine-api] adding namespace I0326 23:14:59.950527 3840 namespace.go:98] [openshift-etcd] adding namespace I0326 23:14:59.950535 3840 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0326 23:14:59.950553 3840 namespace.go:98] [openshift-host-network] adding namespace I0326 23:14:59.950476 3840 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0326 23:14:59.951112 3840 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 623.391µs I0326 23:14:59.951128 3840 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0326 23:14:59.951569 3840 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 436.748µs I0326 23:14:59.951600 3840 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0326 23:14:59.952132 3840 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 519.5µs I0326 23:14:59.952155 3840 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0326 23:14:59.953157 3840 namespace.go:102] [openshift-logging] adding namespace took 2.664032ms I0326 23:14:59.953183 3840 namespace.go:98] [kube-node-lease] adding namespace I0326 23:14:59.953797 3840 namespace.go:102] [openshift-dns-operator] adding namespace took 3.294338ms I0326 23:14:59.953811 3840 namespace.go:98] [openshift-must-gather-operator] adding namespace I0326 23:14:59.954334 3840 namespace.go:102] [openshift-ingress] adding namespace took 3.829151ms I0326 23:14:59.954346 3840 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0326 23:14:59.954923 3840 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 4.413272ms I0326 23:14:59.954937 3840 namespace.go:98] [openshift-node] adding namespace I0326 23:14:59.955456 3840 namespace.go:102] [openshift-backplane-cse] adding namespace took 4.933141ms I0326 23:14:59.955475 3840 namespace.go:98] [openshift-machine-config-operator] adding namespace I0326 23:14:59.955940 3840 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 5.416376ms I0326 23:14:59.955956 3840 namespace.go:98] [openshift-service-ca-operator] adding namespace I0326 23:14:59.956486 3840 namespace.go:102] [openshift] adding namespace took 5.957511ms I0326 23:14:59.956505 3840 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0326 23:14:59.957055 3840 namespace.go:102] [open-cluster-management-2p9mp5cv8t0e1bpnh305hr8ld9shop2c] adding namespace took 6.524488ms I0326 23:14:59.957074 3840 namespace.go:98] [openshift-marketplace] adding namespace I0326 23:14:59.957607 3840 namespace.go:102] [openshift-backplane] adding namespace took 7.075696ms I0326 23:14:59.957619 3840 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0326 23:14:59.958081 3840 namespace.go:102] [openshift-machine-api] adding namespace took 7.545329ms I0326 23:14:59.958097 3840 namespace.go:98] [openshift-monitoring] adding namespace I0326 23:14:59.958657 3840 namespace.go:102] [openshift-etcd] adding namespace took 8.11687ms I0326 23:14:59.958682 3840 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0326 23:14:59.959238 3840 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 8.696935ms I0326 23:14:59.959256 3840 namespace.go:98] [openshift-config-managed] adding namespace I0326 23:14:59.959911 3840 namespace.go:102] [openshift-host-network] adding namespace took 9.346172ms I0326 23:14:59.959928 3840 namespace.go:98] [openshift-image-registry] adding namespace I0326 23:14:59.960438 3840 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 9.82926ms I0326 23:14:59.960453 3840 namespace.go:98] [dedicated-admin] adding namespace I0326 23:14:59.960856 3840 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 8.693849ms I0326 23:14:59.960875 3840 namespace.go:98] [openshift-network-operator] adding namespace I0326 23:14:59.961464 3840 namespace.go:102] [kube-node-lease] adding namespace took 8.272087ms I0326 23:14:59.961479 3840 namespace.go:98] [openshift-route-controller-manager] adding namespace I0326 23:14:59.961949 3840 namespace.go:102] [openshift-must-gather-operator] adding namespace took 8.13092ms I0326 23:14:59.961964 3840 namespace.go:98] [openshift-ingress-canary] adding namespace I0326 23:14:59.962381 3840 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 8.029389ms I0326 23:14:59.962396 3840 namespace.go:98] [openshift-backplane-cee] adding namespace I0326 23:14:59.962866 3840 namespace.go:102] [openshift-node] adding namespace took 7.923199ms I0326 23:14:59.962880 3840 namespace.go:98] [openshift-infra] adding namespace I0326 23:14:59.963312 3840 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.830254ms I0326 23:14:59.963327 3840 namespace.go:98] [openshift-customer-monitoring] adding namespace I0326 23:14:59.963746 3840 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.783936ms I0326 23:14:59.963761 3840 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0326 23:14:59.964161 3840 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.649083ms I0326 23:14:59.964175 3840 namespace.go:98] [openshift-backplane-srep] adding namespace I0326 23:14:59.964580 3840 namespace.go:102] [openshift-marketplace] adding namespace took 7.498971ms I0326 23:14:59.964611 3840 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0326 23:14:59.965202 3840 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.574908ms I0326 23:14:59.965219 3840 namespace.go:98] [openshift-apiserver-operator] adding namespace I0326 23:14:59.965677 3840 namespace.go:102] [openshift-monitoring] adding namespace took 7.571507ms I0326 23:14:59.965696 3840 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0326 23:14:59.966182 3840 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.490778ms I0326 23:14:59.966202 3840 namespace.go:98] [openshift-authentication] adding namespace I0326 23:14:59.966840 3840 namespace.go:102] [openshift-config-managed] adding namespace took 7.577169ms I0326 23:14:59.966856 3840 namespace.go:98] [openshift-controller-manager] adding namespace I0326 23:14:59.967341 3840 namespace.go:102] [openshift-image-registry] adding namespace took 7.406714ms I0326 23:14:59.967353 3840 namespace.go:98] [openshift-insights] adding namespace I0326 23:14:59.967870 3840 namespace.go:102] [dedicated-admin] adding namespace took 7.408562ms I0326 23:14:59.967889 3840 namespace.go:98] [openshift-config-operator] adding namespace I0326 23:14:59.968379 3840 default_node_network_controller.go:518] Node connection status = connected I0326 23:14:59.968379 3840 namespace.go:102] [openshift-network-operator] adding namespace took 7.493899ms I0326 23:14:59.968964 3840 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.480228ms I0326 23:14:59.968979 3840 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0326 23:14:59.969418 3840 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.446631ms I0326 23:14:59.969432 3840 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0326 23:14:59.969903 3840 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.500871ms I0326 23:14:59.969916 3840 namespace.go:98] [openshift-operators-redhat] adding namespace I0326 23:14:59.970366 3840 namespace.go:102] [openshift-infra] adding namespace took 7.47933ms I0326 23:14:59.970383 3840 namespace.go:98] [kube-system] adding namespace I0326 23:14:59.970932 3840 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.598267ms I0326 23:14:59.970949 3840 namespace.go:98] [kube-public] adding namespace I0326 23:14:59.971447 3840 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.680419ms I0326 23:14:59.971463 3840 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0326 23:14:59.971923 3840 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.741935ms I0326 23:14:59.971936 3840 namespace.go:98] [openshift-network-diagnostics] adding namespace I0326 23:14:59.972496 3840 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.878123ms I0326 23:14:59.972512 3840 namespace.go:98] [openshift-console-operator] adding namespace I0326 23:14:59.972981 3840 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.752945ms I0326 23:14:59.972998 3840 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0326 23:14:59.973490 3840 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.785943ms I0326 23:14:59.973510 3840 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0326 23:14:59.974008 3840 namespace.go:102] [openshift-authentication] adding namespace took 7.798586ms I0326 23:14:59.974025 3840 namespace.go:98] [openshift-operators] adding namespace I0326 23:14:59.974548 3840 namespace.go:102] [openshift-controller-manager] adding namespace took 7.686575ms I0326 23:14:59.974563 3840 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0326 23:14:59.975037 3840 namespace.go:102] [openshift-insights] adding namespace took 7.67893ms I0326 23:14:59.975051 3840 namespace.go:98] [openshift-network-node-identity] adding namespace I0326 23:14:59.975464 3840 namespace.go:102] [openshift-config-operator] adding namespace took 7.569658ms I0326 23:14:59.975474 3840 namespace.go:98] [openshift-ingress-operator] adding namespace I0326 23:14:59.975941 3840 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.955944ms I0326 23:14:59.975958 3840 namespace.go:98] [openshift-package-operator] adding namespace I0326 23:14:59.976467 3840 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.029796ms I0326 23:14:59.976480 3840 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace W0326 23:14:59.976888 3840 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-ssx7m: pod kube-system/global-pull-secret-syncer-ssx7m: no pod IPs found W0326 23:14:59.976907 3840 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-tsswl: pod kube-system/global-pull-secret-syncer-tsswl: no pod IPs found I0326 23:14:59.976935 3840 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.012989ms I0326 23:14:59.976951 3840 namespace.go:98] [openshift-console] adding namespace I0326 23:14:59.977470 3840 namespace.go:102] [kube-system] adding namespace took 7.08151ms I0326 23:14:59.977484 3840 namespace.go:98] [openshift-kube-scheduler] adding namespace I0326 23:14:59.977952 3840 namespace.go:102] [kube-public] adding namespace took 6.996827ms I0326 23:14:59.977965 3840 namespace.go:98] [openshift-authentication-operator] adding namespace W0326 23:14:59.978364 3840 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-h6mzl: pod openshift-network-diagnostics/network-check-target-h6mzl: no pod IPs found I0326 23:14:59.978368 3840 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.898575ms W0326 23:14:59.978383 3840 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-jg5zr: pod openshift-network-diagnostics/network-check-target-jg5zr: no pod IPs found I0326 23:14:59.978387 3840 namespace.go:98] [openshift-config] adding namespace W0326 23:14:59.978394 3840 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-vsln9: pod openshift-network-diagnostics/network-check-target-vsln9: no pod IPs found I0326 23:14:59.978976 3840 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.034634ms I0326 23:14:59.978988 3840 namespace.go:98] [openshift-dns] adding namespace I0326 23:14:59.979445 3840 namespace.go:102] [openshift-console-operator] adding namespace took 6.926762ms I0326 23:14:59.979462 3840 namespace.go:98] [openshift-backplane-tam] adding namespace I0326 23:14:59.979978 3840 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.975284ms I0326 23:14:59.979989 3840 namespace.go:98] [openshift-network-console] adding namespace I0326 23:14:59.980486 3840 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.968396ms I0326 23:14:59.980507 3840 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0326 23:14:59.980948 3840 namespace.go:102] [openshift-operators] adding namespace took 6.917527ms I0326 23:14:59.980961 3840 namespace.go:98] [openshift-console-user-settings] adding namespace I0326 23:14:59.981452 3840 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.883422ms I0326 23:14:59.981471 3840 namespace.go:98] [openshift-cluster-version] adding namespace I0326 23:14:59.982048 3840 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.991871ms I0326 23:14:59.982063 3840 namespace.go:98] [openshift-kube-apiserver] adding namespace I0326 23:14:59.982693 3840 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.211876ms I0326 23:14:59.982713 3840 namespace.go:98] [openshift-apiserver] adding namespace I0326 23:14:59.983354 3840 namespace.go:102] [openshift-package-operator] adding namespace took 7.387772ms I0326 23:14:59.984015 3840 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 7.529245ms I0326 23:14:59.984033 3840 namespace.go:98] [openshift-multus] adding namespace I0326 23:14:59.984626 3840 namespace.go:102] [openshift-console] adding namespace took 7.669082ms I0326 23:14:59.985227 3840 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.737073ms I0326 23:14:59.985811 3840 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.837608ms I0326 23:14:59.986349 3840 namespace.go:102] [openshift-config] adding namespace took 7.95493ms I0326 23:14:59.986366 3840 namespace.go:98] [default] adding namespace I0326 23:14:59.986926 3840 namespace.go:102] [openshift-dns] adding namespace took 7.932327ms I0326 23:14:59.987422 3840 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.954779ms I0326 23:14:59.987435 3840 namespace.go:98] [openshift-backplane-csm] adding namespace I0326 23:14:59.987903 3840 namespace.go:102] [openshift-network-console] adding namespace took 7.909559ms I0326 23:14:59.988347 3840 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.83435ms I0326 23:14:59.988810 3840 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.844976ms I0326 23:14:59.989248 3840 namespace.go:102] [openshift-cluster-version] adding namespace took 7.765934ms I0326 23:14:59.989662 3840 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.59357ms I0326 23:14:59.989679 3840 namespace.go:98] [openshift-backplane-mobb] adding namespace W0326 23:14:59.990248 3840 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-9bllr: pod openshift-multus/network-metrics-daemon-9bllr: no pod IPs found W0326 23:14:59.990263 3840 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-c8x7g: pod openshift-multus/network-metrics-daemon-c8x7g: no pod IPs found W0326 23:14:59.990272 3840 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-mqbfm: pod openshift-multus/network-metrics-daemon-mqbfm: no pod IPs found I0326 23:14:59.990282 3840 namespace.go:102] [openshift-apiserver] adding namespace took 7.563757ms I0326 23:14:59.990811 3840 namespace.go:102] [openshift-multus] adding namespace took 6.771927ms I0326 23:14:59.991255 3840 namespace.go:102] [default] adding namespace took 4.884898ms I0326 23:14:59.992322 3840 namespace.go:102] [openshift-backplane-csm] adding namespace took 4.882348ms I0326 23:14:59.992726 3840 namespace.go:102] [openshift-backplane-mobb] adding namespace took 3.041634ms I0326 23:14:59.993113 3840 zone_ic_handler.go:186] Time taken to create transit switch: 292.853µs I0326 23:14:59.993167 3840 master.go:532] Adding or Updating local node "ip-10-0-2-103.ec2.internal" for network "default" I0326 23:14:59.993254 3840 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-167.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-167.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-167.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-167.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-167.ec2.internal I0326 23:14:59.993273 3840 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-99.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-99.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-99.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-99.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-99.ec2.internal W0326 23:15:00.000852 3840 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-jg5zr: pod openshift-network-diagnostics/network-check-target-jg5zr: no pod IPs found W0326 23:15:00.000868 3840 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-mqbfm: pod openshift-multus/network-metrics-daemon-mqbfm: no pod IPs found W0326 23:15:00.000874 3840 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-ssx7m: pod kube-system/global-pull-secret-syncer-ssx7m: no pod IPs found I0326 23:15:00.001813 3840 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-103.ec2.internal for the network default I0326 23:15:00.002501 3840 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0326 23:15:00.002528 3840 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0326 23:15:00.002542 3840 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0326 23:15:00.002571 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-103.ec2.internal in network "default" I0326 23:15:00.002641 3840 node_tracker.go:169] Node ip-10-0-2-103.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:00.002653 3840 services_controller.go:574] Full service sync requested for network=default I0326 23:15:00.002662 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-167.ec2.internal in network "default" I0326 23:15:00.002672 3840 node_tracker.go:235] Node ip-10-0-0-167.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-167.ec2.internal" I0326 23:15:00.002679 3840 node_tracker.go:169] Node ip-10-0-0-167.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:00.002683 3840 services_controller.go:574] Full service sync requested for network=default I0326 23:15:00.002688 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-99.ec2.internal in network "default" I0326 23:15:00.002696 3840 node_tracker.go:235] Node ip-10-0-1-99.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-99.ec2.internal" I0326 23:15:00.002701 3840 node_tracker.go:169] Node ip-10-0-1-99.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:00.002704 3840 services_controller.go:574] Full service sync requested for network=default I0326 23:15:00.103173 3840 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0326 23:15:00.103196 3840 services_controller.go:208] Setting up event handlers for services for network=default I0326 23:15:00.103301 3840 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0326 23:15:00.103336 3840 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0326 23:15:00.103349 3840 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0326 23:15:00.203572 3840 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0326 23:15:00.203607 3840 repair.go:60] Starting repairing loop for services I0326 23:15:00.203704 3840 repair.go:131] Deleted 0 stale service LBs I0326 23:15:00.203723 3840 repair.go:137] Deleted 0 stale Chassis Template Vars I0326 23:15:00.203737 3840 repair.go:62] Finished repairing loop for services: 132.508µs I0326 23:15:00.203762 3840 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0326 23:15:00.203772 3840 services_controller.go:254] Starting workers for network=default I0326 23:15:00.204108 3840 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-h6mzl, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-h6mzl: suppressed error logged: pod openshift-network-diagnostics/network-check-target-h6mzl: no pod IPs found I0326 23:15:00.204136 3840 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-9bllr, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-9bllr: suppressed error logged: pod openshift-multus/network-metrics-daemon-9bllr: no pod IPs found I0326 23:15:00.204169 3840 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-ssx7m] creating logical port kube-system_global-pull-secret-syncer-ssx7m for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:00.204183 3840 obj_retry.go:544] Failed to create *v1.Pod kube-system/global-pull-secret-syncer-tsswl, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-tsswl: suppressed error logged: pod kube-system/global-pull-secret-syncer-tsswl: no pod IPs found I0326 23:15:00.204191 3840 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-jg5zr] creating logical port openshift-network-diagnostics_network-check-target-jg5zr for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:00.204208 3840 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-mqbfm] creating logical port openshift-multus_network-metrics-daemon-mqbfm for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:00.204172 3840 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-c8x7g, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-c8x7g: suppressed error logged: pod openshift-multus/network-metrics-daemon-c8x7g: no pod IPs found I0326 23:15:00.204344 3840 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-mqbfm I0326 23:15:00.204353 3840 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-vsln9, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-vsln9: suppressed error logged: pod openshift-network-diagnostics/network-check-target-vsln9: no pod IPs found I0326 23:15:00.204378 3840 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-ssx7m I0326 23:15:00.204445 3840 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-jg5zr I0326 23:15:00.245197 3840 pod.go:62] [openshift-network-diagnostics/network-check-target-jg5zr] pod update took 40.923412ms I0326 23:15:00.245224 3840 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-jg5zr] addLogicalPort annotation time took 40.955203ms I0326 23:15:00.245494 3840 pod.go:62] [kube-system/global-pull-secret-syncer-ssx7m] pod update took 41.236503ms I0326 23:15:00.245558 3840 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-ssx7m] addLogicalPort annotation time took 41.301151ms I0326 23:15:00.245512 3840 pod.go:62] [openshift-multus/network-metrics-daemon-mqbfm] pod update took 41.241067ms I0326 23:15:00.245615 3840 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-mqbfm] addLogicalPort annotation time took 41.341586ms I0326 23:15:00.246343 3840 pods.go:271] [openshift-network-diagnostics/network-check-target-jg5zr] addLogicalPort took 42.163585ms, libovsdb time 840.408µs I0326 23:15:00.246606 3840 pods.go:271] [kube-system/global-pull-secret-syncer-ssx7m] addLogicalPort took 42.448236ms, libovsdb time 802.797µs I0326 23:15:00.247137 3840 pods.go:271] [openshift-multus/network-metrics-daemon-mqbfm] addLogicalPort took 42.932143ms, libovsdb time 1.231176ms I0326 23:15:00.247832 3840 admin_network_policy_controller.go:227] Starting controller default-network-controller I0326 23:15:00.247852 3840 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0326 23:15:00.247864 3840 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0326 23:15:00.247870 3840 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0326 23:15:00.248067 3840 repair.go:29] Repairing admin network policies took 190.099µs I0326 23:15:00.248166 3840 repair.go:92] Repairing baseline admin network policies took 88.847µs I0326 23:15:00.248734 3840 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0326 23:15:00.248760 3840 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0326 23:15:00.250965 3840 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0326 23:15:00.250992 3840 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0326 23:15:00.251010 3840 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0326 23:15:00.252399 3840 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0326 23:15:00.252421 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0326 23:15:00.252431 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0326 23:15:00.253989 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0326 23:15:00.254009 3840 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0326 23:15:00.254019 3840 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0326 23:15:00.255028 3840 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0326 23:15:00.255046 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0326 23:15:00.255053 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0326 23:15:00.256826 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0326 23:15:00.256851 3840 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0326 23:15:00.256860 3840 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0326 23:15:00.258977 3840 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0326 23:15:00.258995 3840 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0326 23:15:00.259001 3840 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0326 23:15:00.260316 3840 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0326 23:15:00.260335 3840 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0326 23:15:00.260349 3840 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0326 23:15:00.261696 3840 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0326 23:15:00.261721 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0326 23:15:00.261733 3840 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0326 23:15:00.263331 3840 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0326 23:15:00.263352 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0326 23:15:00.263361 3840 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0326 23:15:00.263876 3840 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0326 23:15:00.263892 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0326 23:15:00.264299 3840 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0326 23:15:00.264314 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:15:00.265014 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0326 23:15:00.265035 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0326 23:15:00.265043 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0326 23:15:00.265533 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0326 23:15:00.265551 3840 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0326 23:15:00.265560 3840 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0326 23:15:00.266565 3840 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0326 23:15:00.266600 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0326 23:15:00.266610 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0326 23:15:00.267094 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:15:00.267115 3840 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0326 23:15:00.267125 3840 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0326 23:15:00.267944 3840 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0326 23:15:00.267961 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0326 23:15:00.267970 3840 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0326 23:15:00.268412 3840 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0326 23:15:00.268429 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0326 23:15:00.268435 3840 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0326 23:15:00.268721 3840 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0326 23:15:00.268738 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0326 23:15:00.268744 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0326 23:15:00.270505 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0326 23:15:00.270524 3840 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0326 23:15:00.270533 3840 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0326 23:15:00.271071 3840 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0326 23:15:00.271089 3840 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0326 23:15:00.271097 3840 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0326 23:15:00.271384 3840 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0326 23:15:00.271401 3840 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0326 23:15:00.271411 3840 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0326 23:15:00.273089 3840 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0326 23:15:00.273108 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0326 23:15:00.273120 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0326 23:15:00.273684 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0326 23:15:00.273702 3840 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0326 23:15:00.273711 3840 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0326 23:15:00.274576 3840 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0326 23:15:00.274612 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0326 23:15:00.274620 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0326 23:15:00.275138 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0326 23:15:00.275155 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0326 23:15:00.275163 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0326 23:15:00.275703 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0326 23:15:00.275724 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0326 23:15:00.275746 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0326 23:15:00.276451 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0326 23:15:00.276470 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0326 23:15:00.276478 3840 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0326 23:15:00.277163 3840 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0326 23:15:00.277191 3840 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0326 23:15:00.277199 3840 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0326 23:15:00.277757 3840 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0326 23:15:00.277775 3840 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0326 23:15:00.277788 3840 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0326 23:15:00.278299 3840 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0326 23:15:00.283366 3840 egressfirewall.go:319] Starting EgressFirewall controller I0326 23:15:00.283391 3840 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0326 23:15:00.283427 3840 controller.go:133] Adding controller egress-firewall-controller event handlers I0326 23:15:00.283450 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0326 23:15:00.283468 3840 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0326 23:15:00.283498 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0326 23:15:00.283512 3840 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0326 23:15:00.283796 3840 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0326 23:15:00.283813 3840 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0326 23:15:00.283823 3840 egressqos.go:193] Setting up event handlers for EgressQoS I0326 23:15:00.283900 3840 egressfirewall.go:628] Syncing node "ip-10-0-0-167.ec2.internal" for egress firewall I0326 23:15:00.283916 3840 egressfirewall.go:628] Syncing node "ip-10-0-1-99.ec2.internal" for egress firewall I0326 23:15:00.283923 3840 egressfirewall.go:628] Syncing node "ip-10-0-2-103.ec2.internal" for egress firewall I0326 23:15:00.283972 3840 egressqos.go:245] Starting EgressQoS Controller I0326 23:15:00.283983 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0326 23:15:00.283989 3840 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0326 23:15:00.283994 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0326 23:15:00.283999 3840 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0326 23:15:00.284003 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0326 23:15:00.284007 3840 shared_informer.go:356] "Caches are synced" controller="egressqos" I0326 23:15:00.284011 3840 egressqos.go:259] Repairing EgressQoSes I0326 23:15:00.284015 3840 egressqos.go:399] Starting repairing loop for egressqos I0326 23:15:00.284089 3840 egressqos.go:401] Finished repairing loop for egressqos: 73.187µs I0326 23:15:00.284159 3840 egressservice_zone.go:132] Setting up event handlers for Egress Services I0326 23:15:00.284190 3840 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-103.ec2.internal I0326 23:15:00.284269 3840 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-103.ec2.internal : 84.596µs I0326 23:15:00.284358 3840 egressservice_zone.go:210] Starting Egress Services Controller I0326 23:15:00.284378 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:15:00.284390 3840 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:15:00.284400 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:15:00.284405 3840 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:15:00.284410 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:15:00.284415 3840 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:15:00.284419 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0326 23:15:00.284424 3840 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0326 23:15:00.284427 3840 egressservice_zone.go:228] Repairing Egress Services I0326 23:15:00.284726 3840 master_controller.go:88] Starting Admin Policy Based Route Controller I0326 23:15:00.284738 3840 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:15:00.284774 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-167.ec2.internal I0326 23:15:00.284806 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-167.ec2.internal: 17.498µs I0326 23:15:00.284819 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-99.ec2.internal I0326 23:15:00.284824 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-99.ec2.internal: 6.368µs I0326 23:15:00.284831 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-103.ec2.internal I0326 23:15:00.284835 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-103.ec2.internal: 4.872µs I0326 23:15:00.284852 3840 default_network_controller.go:568] Completing all the Watchers took 334.504417ms I0326 23:15:00.284867 3840 default_network_controller.go:572] Starting unidling controllers I0326 23:15:00.284881 3840 unidle.go:46] Registering OVN SB ControllerEvent handler I0326 23:15:00.284887 3840 unidle.go:63] Populating Initial ContollerEvent events I0326 23:15:00.284899 3840 unidle.go:79] Setting up event handlers for services I0326 23:15:00.284964 3840 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:15:00.469124 3840 default_node_network_controller.go:518] Node connection status = connected I0326 23:15:00.969488 3840 default_node_network_controller.go:518] Node connection status = connected I0326 23:15:01.469597 3840 default_node_network_controller.go:518] Node connection status = connected I0326 23:15:01.969355 3840 default_node_network_controller.go:518] Node connection status = connected I0326 23:15:02.391291 3840 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0326 23:15:02.469104 3840 default_node_network_controller.go:518] Node connection status = connected I0326 23:15:02.483884 3840 bridgeconfig.go:537] Gateway is ready I0326 23:15:02.483902 3840 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0326 23:15:02.493665 3840 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.103. Updating OVN encap IP. I0326 23:15:02.498128 3840 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.103 - it is already configured I0326 23:15:02.498140 3840 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.103/24:{} 172.20.0.1/32:{}]. Updating annotations. I0326 23:15:02.498332 3840 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.103/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-103.ec2.internal","mac-address":"0e:ca:83:9e:a3:07","ip-addresses":["10.0.2.103/24"],"ip-address":"10.0.2.103/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:1162c8fa-9a2c-49fa-b71a-82e1dc57bad2 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.103/24"}] on node ip-10-0-2-103.ec2.internal I0326 23:15:02.515030 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-103.ec2.internal I0326 23:15:02.515048 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-103.ec2.internal: 22.242µs I0326 23:15:02.515099 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-103.ec2.internal in network "default" I0326 23:15:02.515144 3840 egressfirewall.go:628] Syncing node "ip-10-0-2-103.ec2.internal" for egress firewall I0326 23:15:02.515166 3840 master.go:532] Adding or Updating local node "ip-10-0-2-103.ec2.internal" for network "default" I0326 23:15:02.515184 3840 node_tracker.go:169] Node ip-10-0-2-103.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:02.515208 3840 services_controller.go:574] Full service sync requested for network=default I0326 23:15:02.515300 3840 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.103 I0326 23:15:02.515419 3840 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-103.ec2.internal I0326 23:15:02.519305 3840 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-103.ec2.internal. Updating no re-route policies I0326 23:15:02.519386 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-qltrr I0326 23:15:02.519396 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-c8x7g I0326 23:15:02.519405 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-qltrr I0326 23:15:02.519406 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-vbdzq I0326 23:15:02.519411 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-c8x7g I0326 23:15:02.519417 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-vbdzq I0326 23:15:02.519419 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-54sx7 I0326 23:15:02.519432 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-54sx7 I0326 23:15:02.519433 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-vbdzq after 0 failed attempt(s) I0326 23:15:02.519435 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-qltrr after 0 failed attempt(s) I0326 23:15:02.519442 3840 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-c8x7g, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-c8x7g: suppressed error logged: pod openshift-multus/network-metrics-daemon-c8x7g: no pod IPs found I0326 23:15:02.519449 3840 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-103.ec2.internal I0326 23:15:02.519450 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-54sx7 after 0 failed attempt(s) I0326 23:15:02.519452 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-jg5zr I0326 23:15:02.519451 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-jv8nj I0326 23:15:02.519461 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-jg5zr I0326 23:15:02.519463 3840 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-ssx7m I0326 23:15:02.519470 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-jv8nj I0326 23:15:02.519475 3840 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-ssx7m I0326 23:15:02.519491 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-jv8nj after 0 failed attempt(s) I0326 23:15:02.519500 3840 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-ssx7m] creating logical port kube-system_global-pull-secret-syncer-ssx7m for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:02.519484 3840 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-jg5zr] creating logical port openshift-network-diagnostics_network-check-target-jg5zr for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:02.519454 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-103.ec2.internal I0326 23:15:02.519512 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-103.ec2.internal I0326 23:15:02.519521 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-pmhdb I0326 23:15:02.519531 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-pmhdb I0326 23:15:02.519547 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-pmhdb after 0 failed attempt(s) I0326 23:15:02.519564 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-103.ec2.internal after 0 failed attempt(s) I0326 23:15:02.519457 3840 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-103.ec2.internal I0326 23:15:02.519612 3840 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-103.ec2.internal after 0 failed attempt(s) I0326 23:15:02.519845 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-mqbfm I0326 23:15:02.519855 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-97r2t I0326 23:15:02.519865 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-mqbfm I0326 23:15:02.519868 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-jjs7g I0326 23:15:02.519874 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-97r2t I0326 23:15:02.519879 3840 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-wffmz I0326 23:15:02.519887 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-jjs7g I0326 23:15:02.519886 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-vsln9 I0326 23:15:02.519892 3840 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-wffmz I0326 23:15:02.519900 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-vsln9 I0326 23:15:02.519901 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-97r2t after 0 failed attempt(s) I0326 23:15:02.519905 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-jwmwk I0326 23:15:02.519909 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-jjs7g after 0 failed attempt(s) I0326 23:15:02.519914 3840 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-wffmz after 0 failed attempt(s) I0326 23:15:02.519913 3840 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-xplqm I0326 23:15:02.519917 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-h6mzl I0326 23:15:02.519922 3840 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-9bllr I0326 23:15:02.519924 3840 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-xplqm I0326 23:15:02.519929 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-h6mzl I0326 23:15:02.519932 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-9bllr I0326 23:15:02.519938 3840 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-xplqm after 0 failed attempt(s) I0326 23:15:02.519940 3840 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-vsln9, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-vsln9: suppressed error logged: pod openshift-network-diagnostics/network-check-target-vsln9: no pod IPs found I0326 23:15:02.519960 3840 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-h6mzl, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-h6mzl: suppressed error logged: pod openshift-network-diagnostics/network-check-target-h6mzl: no pod IPs found I0326 23:15:02.519903 3840 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-mqbfm] creating logical port openshift-multus_network-metrics-daemon-mqbfm for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:02.519968 3840 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-9bllr, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-9bllr: suppressed error logged: pod openshift-multus/network-metrics-daemon-9bllr: no pod IPs found I0326 23:15:02.519847 3840 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-tsswl I0326 23:15:02.520057 3840 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-tsswl I0326 23:15:02.519919 3840 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-jwmwk I0326 23:15:02.520144 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-jwmwk after 0 failed attempt(s) I0326 23:15:02.520151 3840 obj_retry.go:375] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-tsswl, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-tsswl: suppressed error logged: pod kube-system/global-pull-secret-syncer-tsswl: no pod IPs found I0326 23:15:02.520321 3840 pods.go:271] [kube-system/global-pull-secret-syncer-ssx7m] addLogicalPort took 828.31µs, libovsdb time 299.632µs I0326 23:15:02.520333 3840 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-ssx7m after 0 failed attempt(s) I0326 23:15:02.520750 3840 pods.go:271] [openshift-network-diagnostics/network-check-target-jg5zr] addLogicalPort took 1.269386ms, libovsdb time 471.53µs I0326 23:15:02.520761 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-jg5zr after 0 failed attempt(s) I0326 23:15:02.520889 3840 pods.go:271] [openshift-multus/network-metrics-daemon-mqbfm] addLogicalPort took 993.102µs, libovsdb time 506.218µs I0326 23:15:02.520901 3840 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-mqbfm after 0 failed attempt(s) I0326 23:15:02.533542 3840 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0326 23:15:02.544263 3840 iptables.go:108] Creating table: mangle chain: OUTPUT I0326 23:15:02.547416 3840 iptables.go:108] Creating table: nat chain: OUTPUT I0326 23:15:02.566665 3840 iptables.go:108] Creating table: nat chain: PREROUTING I0326 23:15:02.604572 3840 iptables.go:108] Creating table: filter chain: FORWARD I0326 23:15:02.620713 3840 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0326 23:15:02.620829 3840 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0326 23:15:02.620878 3840 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0326 23:15:02.620958 3840 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0326 23:15:02.621012 3840 gateway_init.go:325] Gateway and management port readiness took 2.655688461s I0326 23:15:02.621040 3840 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0326 23:15:02.621092 3840 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0326 23:15:02.621112 3840 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0326 23:15:02.621123 3840 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0326 23:15:02.621183 3840 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0326 23:15:02.621257 3840 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 45.26µs I0326 23:15:02.621486 3840 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0326 23:15:02.621514 3840 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 18.669µs I0326 23:15:02.621522 3840 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 36.026µs I0326 23:15:02.621530 3840 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0326 23:15:02.621540 3840 nad_controller.go:173] [node-nad-controller NAD controller]: started I0326 23:15:02.621547 3840 default_node_network_controller.go:898] Starting the default node network controller I0326 23:15:02.621553 3840 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0326 23:15:02.621637 3840 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 16.482µs I0326 23:15:02.621742 3840 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0326 23:15:02.621752 3840 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0326 23:15:02.621779 3840 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000aeb750 0xc0002c81c0 0xc0002c8310 0xc0002c8380 0xc0002c83f0 0xc0002c8460 0xc0002c84d0 0xc000c000a0 0xc0002c85b0 0xc0002c8620 0xc0004f0050 0xc00657aee0 0xc0002c8690 map[0x2f67040:0xc003487aa0 0x2f67580:0xc005212de0 0x2f68000:0xc0034879e0 0x2f682a0:0xc003487a40 0x2f687e0:0xc003487b00 0x2f68a80:0xc003487b60 0x2f68d20:0xc005212d80 0x2f69500:0xc005212e40 0x2f697a0:0xc005212ea0 0x2f93d40:0xc000dc73e0 0x2f940e0:0xc000dc7440 0x2f94bc0:0xc000bb1680 0x2f96520:0xc001bec000 0x2f97000:0xc000dd7920 0x2f97e80:0xc000dc7380] 0xc0001191f0 0} I0326 23:15:02.621833 3840 gateway.go:151] Starting gateway service sync I0326 23:15:02.622230 3840 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0326 23:15:02.624290 3840 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0326 23:15:02.633847 3840 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0326 23:15:02.634952 3840 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0326 23:15:02.636001 3840 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0326 23:15:02.669843 3840 gateway.go:168] Gateway service sync done. Time taken: 47.995577ms I0326 23:15:02.670080 3840 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30717 I0326 23:15:02.670716 3840 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000aeb750 0xc0002c81c0 0xc0002c8310 0xc0002c8380 0xc0002c83f0 0xc0002c8460 0xc0002c84d0 0xc000c000a0 0xc0002c85b0 0xc0002c8620 0xc0004f0050 0xc00657aee0 0xc0002c8690 map[0x2f67040:0xc003487aa0 0x2f67580:0xc005212de0 0x2f68000:0xc0034879e0 0x2f682a0:0xc003487a40 0x2f687e0:0xc003487b00 0x2f68a80:0xc003487b60 0x2f68d20:0xc005212d80 0x2f69500:0xc005212e40 0x2f697a0:0xc005212ea0 0x2f93d40:0xc000dc73e0 0x2f940e0:0xc000dc7440 0x2f94bc0:0xc000bb1680 0x2f96520:0xc001bec000 0x2f97000:0xc000dd7920 0x2f97e80:0xc000dc7380] 0xc0001191f0 0} I0326 23:15:02.671142 3840 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000aeb750 0xc0002c81c0 0xc0002c8310 0xc0002c8380 0xc0002c83f0 0xc0002c8460 0xc0002c84d0 0xc000c000a0 0xc0002c85b0 0xc0002c8620 0xc0004f0050 0xc00657aee0 0xc0002c8690 map[0x2f67040:0xc003487aa0 0x2f67580:0xc005212de0 0x2f68000:0xc0034879e0 0x2f682a0:0xc003487a40 0x2f687e0:0xc003487b00 0x2f68a80:0xc003487b60 0x2f68d20:0xc005212d80 0x2f69500:0xc005212e40 0x2f697a0:0xc005212ea0 0x2f93d40:0xc000dc73e0 0x2f940e0:0xc000dc7440 0x2f94bc0:0xc000bb1680 0x2f96520:0xc001bec000 0x2f97000:0xc000dd7920 0x2f97e80:0xc000dc7380] 0xc0001191f0 0} I0326 23:15:02.671177 3840 gateway.go:519] Reconciling gateway with updates I0326 23:15:02.677002 3840 gateway.go:326] Spawning Conntrack Rule Check Thread I0326 23:15:02.677027 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:15:02.677046 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:15:02.677043 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:15:02.677049 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:15:02.677062 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:15:02.677064 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:15:02.677027 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:15:02.677105 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:15:02.677111 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:15:02.677120 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:15:02.677131 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:15:02.677132 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:15:02.677135 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:15:02.677149 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:15:02.677152 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:15:02.677154 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:15:02.677155 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:15:02.677162 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:15:02.677165 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:15:02.677164 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:15:02.677171 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:15:02.677168 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:15:02.677181 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:15:02.677184 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:15:02.677186 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:15:02.677190 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:15:02.677196 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:15:02.677197 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:15:02.677202 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:15:02.677205 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:15:02.677209 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:15:02.677212 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:15:02.677211 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:15:02.677217 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:15:02.677222 3840 default_node_network_controller.go:943] Gateway and management port readiness took 55.461546ms I0326 23:15:02.677222 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:15:02.677171 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:15:02.677234 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:15:02.677231 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:15:02.677208 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:15:02.677245 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:15:02.677250 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:15:02.677176 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:15:02.677260 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:15:02.677211 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:15:02.677269 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:15:02.677269 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:15:02.677150 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:15:02.677283 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:15:02.677280 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:15:02.677296 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:15:02.677294 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:15:02.677306 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:15:02.677160 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:15:02.677208 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:15:02.677316 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:15:02.677318 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:15:02.677282 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:15:02.677321 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:15:02.677329 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:15:02.677333 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:15:02.677337 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:15:02.677344 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:15:02.677342 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:15:02.677346 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:15:02.677249 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:15:02.677356 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:15:02.677357 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:15:02.677362 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:15:02.677362 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:15:02.677367 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:15:02.677373 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:15:02.677344 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:15:02.677382 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:15:02.677318 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:15:02.677392 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:15:02.677404 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:15:02.677172 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:15:02.677362 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:15:02.677238 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:15:02.677253 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:15:02.677255 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:15:02.677259 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:15:02.677519 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:15:02.677171 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:15:02.677538 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:15:02.677273 3840 port_claim.go:71] Svc openshift-ingress/router-default: port 32494 is already open, no action needed I0326 23:15:02.677627 3840 port_claim.go:71] Svc openshift-ingress/router-default: port 31609 is already open, no action needed I0326 23:15:02.677309 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:15:02.677312 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:15:02.677314 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:15:02.677698 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:15:02.677228 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:15:02.677146 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:15:02.677323 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:15:02.677333 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:15:02.677329 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:15:02.677813 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:15:02.677337 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:15:02.677859 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:15:02.677219 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:15:02.677346 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:15:02.677345 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:15:02.677920 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:15:02.677238 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:15:02.677366 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:15:02.677232 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:15:02.677993 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:15:02.677249 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:15:02.678019 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:15:02.678268 3840 node_ip_handler_linux.go:160] Node IP manager is running W0326 23:15:02.714906 3840 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:15:02.715063 3840 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0326 23:15:02.715237 3840 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0326 23:15:02.722827 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:15:02.722858 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:15:02.722869 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:15:02.722881 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:15:02.722892 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:15:02.722904 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:15:02.722921 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:15:02.722931 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:15:02.722939 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:15:02.722956 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:15:02.722968 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:15:02.722975 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:15:02.722981 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:15:02.722987 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:15:02.761415 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:15:02.761426 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:15:02.761450 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:15:02.761459 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:15:02.761464 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:15:02.799160 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:15:02.799183 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:15:02.799191 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:15:02.799202 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:15:02.799211 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:15:02.799216 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:15:04.468190 3840 default_node_network_controller.go:1364] Starting node controller node sync W0326 23:15:04.468222 3840 egressip_healthcheck.go:76] Health checking using insecure connection I0326 23:15:04.468272 3840 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0326 23:15:04.479568 3840 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.361111ms I0326 23:15:04.479786 3840 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0326 23:15:04.480040 3840 egressservice_node.go:93] Setting up event handlers for Egress Services I0326 23:15:04.480138 3840 egressservice_node.go:183] Starting Egress Services Controller I0326 23:15:04.480152 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0326 23:15:04.480161 3840 shared_informer.go:356] "Caches are synced" controller="egressservices" I0326 23:15:04.480168 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0326 23:15:04.480173 3840 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0326 23:15:04.480179 3840 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0326 23:15:04.480183 3840 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0326 23:15:04.480187 3840 egressservice_node.go:197] Repairing Egress Services I0326 23:15:04.489722 3840 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0326 23:15:04.489732 3840 external_controller.go:278] Starting Admin Policy Based Route Controller I0326 23:15:04.489884 3840 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0326 23:15:04.489898 3840 link_network_manager.go:117] Link manager is running I0326 23:15:04.489912 3840 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0326 23:15:04.490051 3840 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0326 23:15:04.490828 3840 vrf_manager.go:128] VRF manager is running I0326 23:15:04.490925 3840 node_controller_manager.go:437] Removing flows to drop GARP I0326 23:15:04.490934 3840 gateway.go:519] Reconciling gateway with updates I0326 23:15:04.511483 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0326 23:15:04.511489 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:15:04.511503 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0326 23:15:04.511505 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0326 23:15:04.511512 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:15:04.511522 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:15:04.511531 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0326 23:15:04.511537 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0326 23:15:04.511552 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0326 23:15:04.511554 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:15:04.511572 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:15:04.511598 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0326 23:15:04.511601 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:15:04.511618 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:15:04.511621 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0326 23:15:04.511625 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0326 23:15:04.511574 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0326 23:15:04.511631 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0326 23:15:04.511600 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:15:04.511641 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0326 23:15:04.511648 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0326 23:15:04.511662 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0326 23:15:04.511663 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0326 23:15:04.511666 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:15:04.511671 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0326 23:15:04.511676 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0326 23:15:04.511680 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:15:04.511647 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0326 23:15:04.511692 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0326 23:15:04.511605 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:15:04.511712 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0326 23:15:04.511683 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0326 23:15:04.511624 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:15:04.511622 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:15:04.511694 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0326 23:15:04.511761 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:15:04.511654 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0326 23:15:04.511572 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0326 23:15:04.511771 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0326 23:15:04.511647 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:15:04.511790 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0326 23:15:04.511611 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:15:04.511797 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0326 23:15:04.511669 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:15:04.511801 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0326 23:15:04.511814 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0326 23:15:04.511740 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:15:04.511845 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0326 23:15:04.511665 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0326 23:15:04.511905 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:15:04.511914 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:15:04.511916 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:15:04.511913 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:15:04.511928 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:15:04.511926 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0326 23:15:04.511930 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:15:04.511944 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0326 23:15:04.511958 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:15:04.511959 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:15:04.511971 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0326 23:15:04.511916 3840 port_claim.go:71] Svc openshift-ingress/router-default: port 32494 is already open, no action needed I0326 23:15:04.511988 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0326 23:15:04.511993 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0326 23:15:04.511999 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:15:04.511970 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:15:04.512015 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:15:04.512021 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0326 23:15:04.512025 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0326 23:15:04.512023 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:15:04.512030 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0326 23:15:04.512031 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:15:04.512027 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:15:04.512037 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:15:04.512042 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:15:04.511982 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:15:04.512047 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0326 23:15:04.512054 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0326 23:15:04.512055 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0326 23:15:04.512063 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0326 23:15:04.512003 3840 port_claim.go:71] Svc openshift-ingress/router-default: port 31609 is already open, no action needed I0326 23:15:04.512005 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0326 23:15:04.512000 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:15:04.512225 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0326 23:15:04.511920 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0326 23:15:04.512298 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0326 23:15:04.512019 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0326 23:15:04.512028 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0326 23:15:04.512350 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0326 23:15:04.512354 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0326 23:15:04.512363 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0326 23:15:04.512368 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:15:04.512016 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:15:04.512379 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0326 23:15:04.512384 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0326 23:15:04.512035 3840 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:15:04.512388 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0326 23:15:04.512391 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0326 23:15:04.512036 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0326 23:15:04.512416 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0326 23:15:04.512040 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0326 23:15:04.512456 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0326 23:15:04.512048 3840 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0326 23:15:04.512465 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0326 23:15:04.548673 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0326 23:15:04.548693 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0326 23:15:04.548699 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0326 23:15:04.548705 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0326 23:15:04.548709 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0326 23:15:04.548715 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0326 23:15:04.548721 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0326 23:15:04.548727 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0326 23:15:04.548732 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0326 23:15:04.548739 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0326 23:15:04.548754 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0326 23:15:04.548763 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0326 23:15:04.548770 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0326 23:15:04.548777 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0326 23:15:04.548781 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0326 23:15:04.548793 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0326 23:15:04.548809 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0326 23:15:04.548815 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0326 23:15:04.548821 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0326 23:15:04.548826 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0326 23:15:04.548831 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0326 23:15:04.584761 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0326 23:15:04.602732 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0326 23:15:04.602787 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0326 23:15:04.602799 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0326 23:15:04.602807 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0326 23:15:04.602812 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0326 23:15:04.602818 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0326 23:15:04.602823 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0326 23:15:04.602829 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0326 23:15:04.602835 3840 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0326 23:15:05.148161 3840 default_network_controller.go:1008] Node "ip-10-0-0-167.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:05.163829 3840 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-xplqm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:15:05.163934 3840 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-xplqm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:15:06.203756 3840 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-xplqm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:15:06.203789 3840 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-xplqm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:15:06.415807 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-103.ec2.internal I0326 23:15:06.415832 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-103.ec2.internal: 29.915µs I0326 23:15:06.443087 3840 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-56599b6895-wp4tk] creating logical port openshift-image-registry_image-registry-56599b6895-wp4tk for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:06.443238 3840 kube.go:256] Updating pod openshift-image-registry/image-registry-56599b6895-wp4tk I0326 23:15:06.467848 3840 pod.go:62] [openshift-image-registry/image-registry-56599b6895-wp4tk] pod update took 24.643418ms I0326 23:15:06.467874 3840 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-56599b6895-wp4tk] addLogicalPort annotation time took 24.671387ms I0326 23:15:06.482482 3840 pods.go:271] [openshift-image-registry/image-registry-56599b6895-wp4tk] addLogicalPort took 39.408497ms, libovsdb time 11.589163ms I0326 23:15:06.482764 3840 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-z86q2] creating logical port openshift-dns_dns-default-z86q2 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:06.482962 3840 kube.go:256] Updating pod openshift-dns/dns-default-z86q2 I0326 23:15:06.483758 3840 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-8dhvq] creating logical port openshift-ingress-canary_ingress-canary-8dhvq for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:06.484001 3840 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-8dhvq I0326 23:15:06.500271 3840 pod.go:62] [openshift-dns/dns-default-z86q2] pod update took 17.357595ms I0326 23:15:06.500289 3840 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-z86q2] addLogicalPort annotation time took 17.380686ms I0326 23:15:06.501340 3840 pods.go:271] [openshift-dns/dns-default-z86q2] addLogicalPort took 18.593452ms, libovsdb time 823.894µs I0326 23:15:06.501547 3840 pod.go:62] [openshift-ingress-canary/ingress-canary-8dhvq] pod update took 17.62956ms I0326 23:15:06.501567 3840 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-8dhvq] addLogicalPort annotation time took 17.686619ms I0326 23:15:06.501793 3840 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-2mh6b] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-2mh6b for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:06.501872 3840 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-2mh6b I0326 23:15:06.502668 3840 pods.go:271] [openshift-ingress-canary/ingress-canary-8dhvq] addLogicalPort took 18.928001ms, libovsdb time 769.842µs I0326 23:15:06.517940 3840 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2mh6b] pod update took 16.09074ms I0326 23:15:06.517960 3840 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-2mh6b] addLogicalPort annotation time took 16.112757ms I0326 23:15:06.518771 3840 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2mh6b] addLogicalPort took 16.988584ms, libovsdb time 607.395µs I0326 23:15:06.851676 3840 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2mh6b 1d0bb4580ebcbe582824572e455bc9795c9f79b9d55c1efdab28beddf4683b80 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2mh6b 1d0bb4580ebcbe582824572e455bc9795c9f79b9d55c1efdab28beddf4683b80 network default NAD default] I0326 23:15:06.854478 3840 cni.go:403] [openshift-ingress-canary/ingress-canary-8dhvq c3850d4187ab223e7e39582144dc809183c4f06f06788f6e4e2489f0e23e2cdc network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-8dhvq c3850d4187ab223e7e39582144dc809183c4f06f06788f6e4e2489f0e23e2cdc network default NAD default] I0326 23:15:06.869952 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-8dhvq, hostIfaceName: c3850d4187ab223, network: default, NAD default, SandboxID: "c3850d4187ab223e7e39582144dc809183c4f06f06788f6e4e2489f0e23e2cdc", PCI device ID: , UID: "b0a42300-9f2c-4adb-92b0-34a8ff9da5b2", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0326 23:15:06.870570 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-2mh6b, hostIfaceName: 1d0bb4580ebcbe5, network: default, NAD default, SandboxID: "1d0bb4580ebcbe582824572e455bc9795c9f79b9d55c1efdab28beddf4683b80", PCI device ID: , UID: "2d30ff2b-1cda-428b-a9a0-660f5066c06c", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0326 23:15:06.913147 3840 cni.go:424] [openshift-ingress-canary/ingress-canary-8dhvq c3850d4187ab223e7e39582144dc809183c4f06f06788f6e4e2489f0e23e2cdc network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-8dhvq c3850d4187ab223e7e39582144dc809183c4f06f06788f6e4e2489f0e23e2cdc network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:e1:53:95:54:03\",\"name\":\"c3850d4187ab223\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f6075cc8-ca15-4418-b69c-3ec2a04b7c19\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:15:06.918234 3840 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2mh6b 1d0bb4580ebcbe582824572e455bc9795c9f79b9d55c1efdab28beddf4683b80 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2mh6b 1d0bb4580ebcbe582824572e455bc9795c9f79b9d55c1efdab28beddf4683b80 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:01:a8:5f:f9:3d\",\"name\":\"1d0bb4580ebcbe5\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e8b06617-cf8e-4687-9d18-4cd45eb3e74c\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:15:08.087170 3840 default_network_controller.go:1008] Node "ip-10-0-1-99.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:11.554639 3840 cni.go:403] [openshift-network-diagnostics/network-check-target-jg5zr 20524552bf086c2c04d0213645d66d665ea0211b8121396eb5ea832486a94607 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-jg5zr 20524552bf086c2c04d0213645d66d665ea0211b8121396eb5ea832486a94607 network default NAD default] I0326 23:15:11.558070 3840 cni.go:403] [kube-system/global-pull-secret-syncer-ssx7m a37106eed5c2d492bb8298313c695ebcb53a863a0016f4441870b2d7a956aad1 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-ssx7m a37106eed5c2d492bb8298313c695ebcb53a863a0016f4441870b2d7a956aad1 network default NAD default] I0326 23:15:11.563671 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-jg5zr, hostIfaceName: 20524552bf086c2, network: default, NAD default, SandboxID: "20524552bf086c2c04d0213645d66d665ea0211b8121396eb5ea832486a94607", PCI device ID: , UID: "1d40179f-5ece-45c7-b525-36ca45c8e212", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0326 23:15:11.569396 3840 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-ssx7m, hostIfaceName: a37106eed5c2d49, network: default, NAD default, SandboxID: "a37106eed5c2d492bb8298313c695ebcb53a863a0016f4441870b2d7a956aad1", PCI device ID: , UID: "3dc7a553-8afe-431b-a7a1-69139c29dee9", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0326 23:15:11.609114 3840 cni.go:424] [openshift-network-diagnostics/network-check-target-jg5zr 20524552bf086c2c04d0213645d66d665ea0211b8121396eb5ea832486a94607 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-jg5zr 20524552bf086c2c04d0213645d66d665ea0211b8121396eb5ea832486a94607 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:01:01:c6:56:d3\",\"name\":\"20524552bf086c2\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c12a9738-50c0-468a-8a58-7d67b36e4aa7\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:15:11.624906 3840 cni.go:424] [kube-system/global-pull-secret-syncer-ssx7m a37106eed5c2d492bb8298313c695ebcb53a863a0016f4441870b2d7a956aad1 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-ssx7m a37106eed5c2d492bb8298313c695ebcb53a863a0016f4441870b2d7a956aad1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:bb:b6:3c:e3:5a\",\"name\":\"a37106eed5c2d49\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c63fc64b-c134-4094-b154-6b46d4889dc5\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:15:12.182565 3840 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:15:14.625335 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-167.ec2.internal I0326 23:15:14.625358 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-167.ec2.internal: 30.766µs I0326 23:15:14.625437 3840 egressfirewall.go:628] Syncing node "ip-10-0-0-167.ec2.internal" for egress firewall I0326 23:15:14.625475 3840 default_network_controller.go:1008] Node "ip-10-0-0-167.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0326 23:15:17.205142 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:15:17.228396 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0326 23:15:17.585234 3840 egressfirewall.go:628] Syncing node "ip-10-0-1-99.ec2.internal" for egress firewall I0326 23:15:17.585256 3840 default_network_controller.go:1008] Node "ip-10-0-1-99.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:17.585240 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-99.ec2.internal I0326 23:15:17.585326 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-99.ec2.internal: 89.064µs I0326 23:15:18.258126 3840 default_network_controller.go:1008] Node "ip-10-0-1-99.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:18.273874 3840 default_network_controller.go:1008] Node "ip-10-0-0-167.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:20.731983 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-167.ec2.internal in network "default" I0326 23:15:20.732007 3840 default_network_controller.go:1008] Node "ip-10-0-0-167.ec2.internal" in remote zone "ip-10-0-0-167.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:20.732034 3840 node_tracker.go:235] Node ip-10-0-0-167.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-167.ec2.internal" I0326 23:15:20.732050 3840 node_tracker.go:169] Node ip-10-0-0-167.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:20.732064 3840 services_controller.go:574] Full service sync requested for network=default W0326 23:15:20.732636 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:15:20.824850 3840 default_network_controller.go:1008] Node "ip-10-0-0-167.ec2.internal" in remote zone "ip-10-0-0-167.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:20.848387 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-167.ec2.internal in network "default" I0326 23:15:20.848402 3840 default_network_controller.go:1008] Node "ip-10-0-0-167.ec2.internal" in remote zone "ip-10-0-0-167.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:20.848436 3840 node_tracker.go:169] Node ip-10-0-0-167.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:20.848448 3840 services_controller.go:574] Full service sync requested for network=default W0326 23:15:20.848678 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:15:20.849227 3840 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-167.ec2.internal for the network default I0326 23:15:20.850499 3840 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-167.ec2.internal" on network "default" took: 1.287274ms I0326 23:15:22.401890 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-167.ec2.internal I0326 23:15:22.401913 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-167.ec2.internal: 28.621µs I0326 23:15:22.401959 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-167.ec2.internal in network "default" I0326 23:15:22.401979 3840 egressfirewall.go:628] Syncing node "ip-10-0-0-167.ec2.internal" for egress firewall I0326 23:15:22.402008 3840 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-167.ec2.internal", IP: 10.0.0.167 to PMTUD blocking rules I0326 23:15:22.402026 3840 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-167.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:15:22.402013 3840 node_tracker.go:169] Node ip-10-0-0-167.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:22.402129 3840 services_controller.go:574] Full service sync requested for network=default W0326 23:15:22.402524 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:15:22.408105 3840 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-167.ec2.internal. Updating no re-route policies I0326 23:15:22.416929 3840 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-167.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:15:22.416946 3840 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-167.ec2.internal", IP: 10.0.0.167 to PMTUD blocking rules I0326 23:15:23.083911 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-99.ec2.internal in network "default" I0326 23:15:23.083939 3840 default_network_controller.go:1008] Node "ip-10-0-1-99.ec2.internal" in remote zone "ip-10-0-1-99.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:23.083944 3840 node_tracker.go:235] Node ip-10-0-1-99.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-99.ec2.internal" I0326 23:15:23.083965 3840 node_tracker.go:169] Node ip-10-0-1-99.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:23.083984 3840 services_controller.go:574] Full service sync requested for network=default W0326 23:15:23.084544 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:15:23.178187 3840 default_network_controller.go:1008] Node "ip-10-0-1-99.ec2.internal" in remote zone "ip-10-0-1-99.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0326 23:15:23.203402 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-99.ec2.internal in network "default" I0326 23:15:23.203464 3840 node_tracker.go:169] Node ip-10-0-1-99.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:23.203481 3840 services_controller.go:574] Full service sync requested for network=default I0326 23:15:23.203485 3840 default_network_controller.go:1008] Node "ip-10-0-1-99.ec2.internal" in remote zone "ip-10-0-1-99.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0326 23:15:23.203683 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:15:23.204180 3840 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-99.ec2.internal for the network default I0326 23:15:23.205494 3840 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-99.ec2.internal" on network "default" took: 1.341851ms I0326 23:15:25.254281 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-99.ec2.internal I0326 23:15:25.254309 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-99.ec2.internal: 36.562µs I0326 23:15:25.254364 3840 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-99.ec2.internal in network "default" I0326 23:15:25.254373 3840 egressfirewall.go:628] Syncing node "ip-10-0-1-99.ec2.internal" for egress firewall I0326 23:15:25.254414 3840 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-99.ec2.internal", IP: 10.0.1.99 to PMTUD blocking rules I0326 23:15:25.254430 3840 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-99.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:15:25.254422 3840 node_tracker.go:169] Node ip-10-0-1-99.ec2.internal switch + router changed, syncing services in network "default" I0326 23:15:25.254506 3840 services_controller.go:574] Full service sync requested for network=default W0326 23:15:25.254678 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:15:25.260756 3840 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-99.ec2.internal. Updating no re-route policies I0326 23:15:25.272112 3840 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-99.ec2.internal", IP: 10.0.1.99 to PMTUD blocking rules I0326 23:15:25.272132 3840 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-99.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0326 23:15:25.994292 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-167.ec2.internal I0326 23:15:25.994323 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-167.ec2.internal: 39.113µs I0326 23:15:28.979455 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-99.ec2.internal I0326 23:15:28.979486 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-99.ec2.internal: 37.357µs I0326 23:15:32.318805 3840 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-b2fb9] creating logical port openshift-service-ca-operator_service-ca-operator-7b89bc695f-b2fb9 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:32.318876 3840 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-7b89bc695f-b2fb9 I0326 23:15:32.328557 3840 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-wj8cq] creating logical port openshift-console-operator_console-operator-8777bd9bb-wj8cq for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:32.328651 3840 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-wj8cq I0326 23:15:32.337662 3840 pod.go:62] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-b2fb9] pod update took 18.803344ms I0326 23:15:32.337680 3840 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-b2fb9] addLogicalPort annotation time took 18.825274ms I0326 23:15:32.338703 3840 pods.go:271] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-b2fb9] addLogicalPort took 19.91241ms, libovsdb time 786.871µs I0326 23:15:32.351947 3840 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-wj8cq] pod update took 23.317307ms I0326 23:15:32.351968 3840 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-wj8cq] addLogicalPort annotation time took 23.34204ms I0326 23:15:32.352994 3840 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-wj8cq] addLogicalPort took 24.449408ms, libovsdb time 755.652µs I0326 23:15:32.434888 3840 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-6565674565-gspwq] creating logical port openshift-ingress_router-default-6565674565-gspwq for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:32.434971 3840 kube.go:256] Updating pod openshift-ingress/router-default-6565674565-gspwq I0326 23:15:32.435249 3840 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7c965c98cf-ts7zq] creating logical port openshift-ingress_router-default-7c965c98cf-ts7zq for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:32.435312 3840 kube.go:256] Updating pod openshift-ingress/router-default-7c965c98cf-ts7zq I0326 23:15:32.466808 3840 pod.go:62] [openshift-ingress/router-default-6565674565-gspwq] pod update took 31.860561ms I0326 23:15:32.466831 3840 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-6565674565-gspwq] addLogicalPort annotation time took 31.885979ms I0326 23:15:32.467180 3840 pod.go:62] [openshift-ingress/router-default-7c965c98cf-ts7zq] pod update took 31.887448ms I0326 23:15:32.467203 3840 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-7c965c98cf-ts7zq] addLogicalPort annotation time took 31.912823ms I0326 23:15:32.467982 3840 pods.go:271] [openshift-ingress/router-default-6565674565-gspwq] addLogicalPort took 33.103001ms, libovsdb time 736.783µs I0326 23:15:32.468242 3840 pods.go:271] [openshift-ingress/router-default-7c965c98cf-ts7zq] addLogicalPort took 33.020206ms, libovsdb time 754.275µs I0326 23:15:32.712142 3840 cni.go:403] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-b2fb9 a8c4f00f458c77f254e536186cbd733d40d8b0e3ea938a3226ef1de73523c7c7 network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-b2fb9 a8c4f00f458c77f254e536186cbd733d40d8b0e3ea938a3226ef1de73523c7c7 network default NAD default] I0326 23:15:32.721963 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-7b89bc695f-b2fb9, hostIfaceName: a8c4f00f458c77f, network: default, NAD default, SandboxID: "a8c4f00f458c77f254e536186cbd733d40d8b0e3ea938a3226ef1de73523c7c7", PCI device ID: , UID: "f516d532-05ec-4aaf-a542-f80ac6eac828", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0326 23:15:32.733732 3840 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-wj8cq ff8bcf3e5e3f39bc0b8fd7e6bea94ac4287edf98e3a0851328ff3613bccae6b8 network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-wj8cq ff8bcf3e5e3f39bc0b8fd7e6bea94ac4287edf98e3a0851328ff3613bccae6b8 network default NAD default] I0326 23:15:32.743171 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-wj8cq, hostIfaceName: ff8bcf3e5e3f39b, network: default, NAD default, SandboxID: "ff8bcf3e5e3f39bc0b8fd7e6bea94ac4287edf98e3a0851328ff3613bccae6b8", PCI device ID: , UID: "f74b62e4-3e64-461b-a363-e4a29e9e72c1", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0326 23:15:32.764130 3840 cni.go:424] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-b2fb9 a8c4f00f458c77f254e536186cbd733d40d8b0e3ea938a3226ef1de73523c7c7 network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-b2fb9 a8c4f00f458c77f254e536186cbd733d40d8b0e3ea938a3226ef1de73523c7c7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:4b:7d:6b:a9:5f\",\"name\":\"a8c4f00f458c77f\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ed8eff0f-809d-4426-8ba3-313b25c563e2\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:15:32.778834 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:15:32.781970 3840 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-wj8cq ff8bcf3e5e3f39bc0b8fd7e6bea94ac4287edf98e3a0851328ff3613bccae6b8 network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-wj8cq ff8bcf3e5e3f39bc0b8fd7e6bea94ac4287edf98e3a0851328ff3613bccae6b8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:ed:5f:bf:ff:4b\",\"name\":\"ff8bcf3e5e3f39b\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c338f763-360f-40cf-9327-d5b24e879bd2\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:15:35.760550 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0326 23:15:36.255071 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:15:38.190251 3840 namespace.go:98] [openshift-service-ca] adding namespace I0326 23:15:38.191194 3840 namespace.go:102] [openshift-service-ca] adding namespace took 919.151µs I0326 23:15:38.205396 3840 namespace.go:142] [openshift-service-ca] updating namespace I0326 23:15:38.304176 3840 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0326 23:15:38.304944 3840 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 749.285µs I0326 23:15:38.326829 3840 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0326 23:15:38.437663 3840 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dc774] creating logical port openshift-kube-storage-version-migrator_migrator-854b5fb5c9-dc774 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:15:38.437748 3840 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dc774 I0326 23:15:38.454548 3840 pod.go:62] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dc774] pod update took 16.83041ms I0326 23:15:38.454570 3840 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dc774] addLogicalPort annotation time took 16.856286ms I0326 23:15:38.455578 3840 pods.go:271] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dc774] addLogicalPort took 17.925592ms, libovsdb time 708.246µs I0326 23:15:38.829618 3840 cni.go:403] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dc774 f2ee726ccef30958a32bdc3fc675651cf78c4e957c2ced087f1d337c93c0fff3 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dc774 f2ee726ccef30958a32bdc3fc675651cf78c4e957c2ced087f1d337c93c0fff3 network default NAD default] I0326 23:15:38.838702 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-854b5fb5c9-dc774, hostIfaceName: f2ee726ccef3095, network: default, NAD default, SandboxID: "f2ee726ccef30958a32bdc3fc675651cf78c4e957c2ced087f1d337c93c0fff3", PCI device ID: , UID: "ea12a06a-351c-4be8-9612-a7e9107fb128", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0326 23:15:38.886240 3840 cni.go:424] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dc774 f2ee726ccef30958a32bdc3fc675651cf78c4e957c2ced087f1d337c93c0fff3 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-dc774 f2ee726ccef30958a32bdc3fc675651cf78c4e957c2ced087f1d337c93c0fff3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:ab:81:a3:31:64\",\"name\":\"f2ee726ccef3095\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/17e2afd2-c4cd-4691-9f96-7cb92e9023cc\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:15:47.289721 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0326 23:15:59.964782 3840 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:15:59.967725 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0326 23:16:00.022851 3840 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-2vwbz] creating logical port openshift-insights_insights-runtime-extractor-2vwbz for pod on switch ip-10-0-2-103.ec2.internal I0326 23:16:00.022933 3840 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-2vwbz I0326 23:16:00.056575 3840 pod.go:62] [openshift-insights/insights-runtime-extractor-2vwbz] pod update took 33.667195ms I0326 23:16:00.056609 3840 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-2vwbz] addLogicalPort annotation time took 33.704327ms I0326 23:16:00.057576 3840 pods.go:271] [openshift-insights/insights-runtime-extractor-2vwbz] addLogicalPort took 34.73674ms, libovsdb time 698.747µs I0326 23:16:00.377312 3840 cni.go:403] [openshift-insights/insights-runtime-extractor-2vwbz 7eb2f44d378d776aa688fdbb887d65e8f3060ea67f815d485ca2898015dda418 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-2vwbz 7eb2f44d378d776aa688fdbb887d65e8f3060ea67f815d485ca2898015dda418 network default NAD default] I0326 23:16:00.385620 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-2vwbz, hostIfaceName: 7eb2f44d378d776, network: default, NAD default, SandboxID: "7eb2f44d378d776aa688fdbb887d65e8f3060ea67f815d485ca2898015dda418", PCI device ID: , UID: "ed35fb52-d93e-417e-9dfe-14ff04898442", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0326 23:16:00.425202 3840 cni.go:424] [openshift-insights/insights-runtime-extractor-2vwbz 7eb2f44d378d776aa688fdbb887d65e8f3060ea67f815d485ca2898015dda418 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-2vwbz 7eb2f44d378d776aa688fdbb887d65e8f3060ea67f815d485ca2898015dda418 network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:e2:3f:a8:65:6e\",\"name\":\"7eb2f44d378d776\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e13549e0-46cf-47b3-8ea7-fbdfbe70f178\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:16:00.907977 3840 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:00.907998 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:00.908004 3840 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0326 23:16:01.347980 3840 obj_retry.go:464] Detected object openshift-image-registry/image-registry-56599b6895-wp4tk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:16:01.348019 3840 pods.go:176] Deleting pod: openshift-image-registry/image-registry-56599b6895-wp4tk I0326 23:16:01.349199 3840 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-56599b6895-wp4tk, ips: 10.128.0.6 I0326 23:16:01.349223 3840 obj_retry.go:464] Detected object openshift-image-registry/image-registry-56599b6895-wp4tk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:16:01.359699 3840 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6ddfbd4cc8-5ckz2] creating logical port openshift-image-registry_image-registry-6ddfbd4cc8-5ckz2 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:16:01.359807 3840 kube.go:256] Updating pod openshift-image-registry/image-registry-6ddfbd4cc8-5ckz2 I0326 23:16:01.360533 3840 obj_retry.go:458] Detected object openshift-image-registry/image-registry-56599b6895-wp4tk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:01.360552 3840 obj_retry.go:458] Detected object openshift-image-registry/image-registry-56599b6895-wp4tk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:01.366542 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-56599b6895-wp4tk I0326 23:16:01.366558 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-56599b6895-wp4tk I0326 23:16:01.378090 3840 pod.go:62] [openshift-image-registry/image-registry-6ddfbd4cc8-5ckz2] pod update took 18.306327ms I0326 23:16:01.378111 3840 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-6ddfbd4cc8-5ckz2] addLogicalPort annotation time took 18.329986ms I0326 23:16:01.379115 3840 pods.go:271] [openshift-image-registry/image-registry-6ddfbd4cc8-5ckz2] addLogicalPort took 19.429241ms, libovsdb time 741.612µs I0326 23:16:01.714891 3840 cni.go:403] [openshift-image-registry/image-registry-6ddfbd4cc8-5ckz2 e74d78e16bcf0711fe18c52035f35db16206a9b52f076a31276a64addc7e143f network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-6ddfbd4cc8-5ckz2 e74d78e16bcf0711fe18c52035f35db16206a9b52f076a31276a64addc7e143f network default NAD default] I0326 23:16:01.723542 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6ddfbd4cc8-5ckz2, hostIfaceName: e74d78e16bcf071, network: default, NAD default, SandboxID: "e74d78e16bcf0711fe18c52035f35db16206a9b52f076a31276a64addc7e143f", PCI device ID: , UID: "014f875b-acef-4716-938f-0736642c7188", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0326 23:16:01.763080 3840 cni.go:424] [openshift-image-registry/image-registry-6ddfbd4cc8-5ckz2 e74d78e16bcf0711fe18c52035f35db16206a9b52f076a31276a64addc7e143f network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-6ddfbd4cc8-5ckz2 e74d78e16bcf0711fe18c52035f35db16206a9b52f076a31276a64addc7e143f network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:66:19:3b:d3:e5\",\"name\":\"e74d78e16bcf071\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/435d9e25-4991-40f1-a235-1c339010b06a\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:16:01.818866 3840 obj_retry.go:464] Detected object openshift-image-registry/image-registry-574cbc9d5b-54cdm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:16:01.819417 3840 obj_retry.go:464] Detected object openshift-image-registry/image-registry-574cbc9d5b-54cdm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:16:01.833537 3840 obj_retry.go:458] Detected object openshift-image-registry/image-registry-574cbc9d5b-54cdm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:01.833551 3840 obj_retry.go:458] Detected object openshift-image-registry/image-registry-574cbc9d5b-54cdm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:01.839279 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-574cbc9d5b-54cdm I0326 23:16:01.839304 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-574cbc9d5b-54cdm W0326 23:16:02.354978 3840 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:16:02.825602 3840 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0326 23:16:04.338185 3840 cni.go:403] [openshift-ingress/router-default-6565674565-gspwq 6f5d2bc6ad8ce1d4a4459598408cc5dc6f739a48929fde9848a069651628cac4 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-6565674565-gspwq 6f5d2bc6ad8ce1d4a4459598408cc5dc6f739a48929fde9848a069651628cac4 network default NAD default] I0326 23:16:04.346743 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-6565674565-gspwq, hostIfaceName: 6f5d2bc6ad8ce1d, network: default, NAD default, SandboxID: "6f5d2bc6ad8ce1d4a4459598408cc5dc6f739a48929fde9848a069651628cac4", PCI device ID: , UID: "8a8db327-d109-4ce7-838a-95f9cdf18b07", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0326 23:16:04.387482 3840 cni.go:424] [openshift-ingress/router-default-6565674565-gspwq 6f5d2bc6ad8ce1d4a4459598408cc5dc6f739a48929fde9848a069651628cac4 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-6565674565-gspwq 6f5d2bc6ad8ce1d4a4459598408cc5dc6f739a48929fde9848a069651628cac4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:96:0b:b2:e6:12\",\"name\":\"6f5d2bc6ad8ce1d\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c4ad04ef-3418-4e14-a466-b45e00f6b61a\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:16:04.634744 3840 cni.go:403] [openshift-ingress/router-default-7c965c98cf-ts7zq 096804449a99fbabe646637205cdeb5cbeadbea6f1f473759d9b05c8a28b4091 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-7c965c98cf-ts7zq 096804449a99fbabe646637205cdeb5cbeadbea6f1f473759d9b05c8a28b4091 network default NAD default] I0326 23:16:04.642920 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7c965c98cf-ts7zq, hostIfaceName: 096804449a99fba, network: default, NAD default, SandboxID: "096804449a99fbabe646637205cdeb5cbeadbea6f1f473759d9b05c8a28b4091", PCI device ID: , UID: "92b5a313-3e07-4ebd-8223-3c22b1ffa2b2", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0326 23:16:04.683375 3840 cni.go:424] [openshift-ingress/router-default-7c965c98cf-ts7zq 096804449a99fbabe646637205cdeb5cbeadbea6f1f473759d9b05c8a28b4091 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-7c965c98cf-ts7zq 096804449a99fbabe646637205cdeb5cbeadbea6f1f473759d9b05c8a28b4091 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:13:94:4e:38:eb\",\"name\":\"096804449a99fba\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/58498ff4-3039-47ae-b7ef-8c37631103aa\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:16:04.833290 3840 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:04.833312 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:04.833318 3840 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:05.372080 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:16:05.372107 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0326 23:16:05.373597 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:05.373613 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:05.373618 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:05.403538 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:05.403552 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:05.403559 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:05.403634 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:16:05.403650 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0326 23:16:05.874048 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0326 23:16:05.874067 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0326 23:16:05.874926 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:05.874944 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:05.874949 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:06.379996 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:06.380017 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:06.380024 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0326 23:16:06.383116 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0326 23:16:06.383136 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0326 23:16:06.493377 3840 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:16:06.594414 3840 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0326 23:16:09.858831 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0326 23:16:10.454124 3840 cni.go:403] [openshift-dns/dns-default-z86q2 7dd84d5bf234bab6326ede237b6d31ab6edb4db640ad08029afdb635b55b0670 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-z86q2 7dd84d5bf234bab6326ede237b6d31ab6edb4db640ad08029afdb635b55b0670 network default NAD default] I0326 23:16:10.462312 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-z86q2, hostIfaceName: 7dd84d5bf234bab, network: default, NAD default, SandboxID: "7dd84d5bf234bab6326ede237b6d31ab6edb4db640ad08029afdb635b55b0670", PCI device ID: , UID: "f00c8b9b-4086-4b37-8978-2c525aec2427", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0326 23:16:10.502567 3840 cni.go:424] [openshift-dns/dns-default-z86q2 7dd84d5bf234bab6326ede237b6d31ab6edb4db640ad08029afdb635b55b0670 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-z86q2 7dd84d5bf234bab6326ede237b6d31ab6edb4db640ad08029afdb635b55b0670 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:a2:20:71:86:15\",\"name\":\"7dd84d5bf234bab\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b14c0f63-f25c-4dc5-a002-cf9012ab21cc\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:16:14.844153 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:14.844176 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:14.844183 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:14.844199 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:14.844204 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:14.844209 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0326 23:16:17.085040 3840 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390j2jb4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:16:17.085582 3840 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390j2jb4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:16:17.085634 3840 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390j2jb4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:16:17.085643 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0326 23:16:17.085675 3840 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390j2jb4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:16:17.085680 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0326 23:16:17.456874 3840 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:16:17.457005 3840 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0326 23:16:17.485337 3840 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 28.381424ms I0326 23:16:17.485360 3840 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 28.408859ms I0326 23:16:17.486534 3840 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 29.67751ms, libovsdb time 840.923µs I0326 23:16:18.124105 3840 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390j2jb4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:18.124129 3840 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390j2jb4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:18.124148 3840 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390j2jb4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:18.124161 3840 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390j2jb4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:18.418374 3840 cni.go:403] [openshift-monitoring/alertmanager-main-1 45ff77f32a0b402a2a21c7e07257b80d7f3a710f71f1e8cf4b05f7c1d86a9e0b network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 45ff77f32a0b402a2a21c7e07257b80d7f3a710f71f1e8cf4b05f7c1d86a9e0b network default NAD default] I0326 23:16:18.427461 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 45ff77f32a0b402, network: default, NAD default, SandboxID: "45ff77f32a0b402a2a21c7e07257b80d7f3a710f71f1e8cf4b05f7c1d86a9e0b", PCI device ID: , UID: "db725054-5c70-40b1-8982-8990a5e96efb", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0326 23:16:18.470721 3840 cni.go:424] [openshift-monitoring/alertmanager-main-1 45ff77f32a0b402a2a21c7e07257b80d7f3a710f71f1e8cf4b05f7c1d86a9e0b network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 45ff77f32a0b402a2a21c7e07257b80d7f3a710f71f1e8cf4b05f7c1d86a9e0b network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:0f:64:48:7e:77\",\"name\":\"45ff77f32a0b402\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7e56ddb1-bfe7-478c-a977-6f0a3e81f589\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:16:18.541310 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0326 23:16:18.744675 3840 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-j5p49] creating logical port openshift-console_downloads-56d45c498b-j5p49 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:16:18.744796 3840 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-j5p49 I0326 23:16:18.763967 3840 pod.go:62] [openshift-console/downloads-56d45c498b-j5p49] pod update took 19.198182ms I0326 23:16:18.763988 3840 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-j5p49] addLogicalPort annotation time took 19.222818ms I0326 23:16:18.765165 3840 pods.go:271] [openshift-console/downloads-56d45c498b-j5p49] addLogicalPort took 20.511339ms, libovsdb time 853.408µs I0326 23:16:19.102253 3840 cni.go:403] [openshift-console/downloads-56d45c498b-j5p49 c8911d3eef9b87cf1da2175b3d18194a7330844b09eca792d3bceb1fbf9c603a network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-j5p49 c8911d3eef9b87cf1da2175b3d18194a7330844b09eca792d3bceb1fbf9c603a network default NAD default] I0326 23:16:19.112077 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-j5p49, hostIfaceName: c8911d3eef9b87c, network: default, NAD default, SandboxID: "c8911d3eef9b87cf1da2175b3d18194a7330844b09eca792d3bceb1fbf9c603a", PCI device ID: , UID: "e1ac8ce5-443f-403a-98cb-b3c3f15330e6", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0326 23:16:19.156027 3840 cni.go:424] [openshift-console/downloads-56d45c498b-j5p49 c8911d3eef9b87cf1da2175b3d18194a7330844b09eca792d3bceb1fbf9c603a network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-j5p49 c8911d3eef9b87cf1da2175b3d18194a7330844b09eca792d3bceb1fbf9c603a network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:7b:a8:a7:c2:bc\",\"name\":\"c8911d3eef9b87c\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/80824903-3e14-4606-8a77-8a4fd45f5575\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:16:19.885893 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:16:19.885918 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:16:19.956139 3840 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:19.956165 3840 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:19.956173 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:20.467169 3840 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:20.467195 3840 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:20.467202 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:16:20.881829 3840 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-956879f57-ppbxx] creating logical port openshift-monitoring_metrics-server-956879f57-ppbxx for pod on switch ip-10-0-2-103.ec2.internal I0326 23:16:20.881959 3840 kube.go:256] Updating pod openshift-monitoring/metrics-server-956879f57-ppbxx I0326 23:16:20.902102 3840 pod.go:62] [openshift-monitoring/metrics-server-956879f57-ppbxx] pod update took 20.165759ms I0326 23:16:20.902126 3840 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-956879f57-ppbxx] addLogicalPort annotation time took 20.193119ms I0326 23:16:20.903298 3840 pods.go:271] [openshift-monitoring/metrics-server-956879f57-ppbxx] addLogicalPort took 21.48608ms, libovsdb time 833.76µs I0326 23:16:21.095250 3840 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-2rblw] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-2rblw for pod on switch ip-10-0-2-103.ec2.internal I0326 23:16:21.095375 3840 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-2rblw I0326 23:16:21.113697 3840 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-2rblw] pod update took 18.341988ms I0326 23:16:21.113722 3840 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-2rblw] addLogicalPort annotation time took 18.370627ms I0326 23:16:21.114986 3840 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-2rblw] addLogicalPort took 19.755558ms, libovsdb time 961.38µs I0326 23:16:21.250838 3840 cni.go:403] [openshift-monitoring/metrics-server-956879f57-ppbxx 031b8acdfe3a91ff630557925378e02796b3b7c6b37d9e637f3e467adbf8f9cd network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-956879f57-ppbxx 031b8acdfe3a91ff630557925378e02796b3b7c6b37d9e637f3e467adbf8f9cd network default NAD default] I0326 23:16:21.261180 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-956879f57-ppbxx, hostIfaceName: 031b8acdfe3a91f, network: default, NAD default, SandboxID: "031b8acdfe3a91ff630557925378e02796b3b7c6b37d9e637f3e467adbf8f9cd", PCI device ID: , UID: "6c58378f-2665-449f-acca-3557b4ff9133", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0326 23:16:21.305762 3840 cni.go:424] [openshift-monitoring/metrics-server-956879f57-ppbxx 031b8acdfe3a91ff630557925378e02796b3b7c6b37d9e637f3e467adbf8f9cd network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-956879f57-ppbxx 031b8acdfe3a91ff630557925378e02796b3b7c6b37d9e637f3e467adbf8f9cd network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:a6:46:7f:d0:ac\",\"name\":\"031b8acdfe3a91f\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1134d581-940f-470f-acc4-6a36bcff819b\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:16:21.623642 3840 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-2rblw 7e1dad10171f9ac631e248b5027c5b39e2cb48d710e50335b181de870861e44c network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-2rblw 7e1dad10171f9ac631e248b5027c5b39e2cb48d710e50335b181de870861e44c network default NAD default] I0326 23:16:21.633235 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-2rblw, hostIfaceName: 7e1dad10171f9ac, network: default, NAD default, SandboxID: "7e1dad10171f9ac631e248b5027c5b39e2cb48d710e50335b181de870861e44c", PCI device ID: , UID: "b4d7ee91-16fa-4c80-86bc-35ec118e5489", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0326 23:16:21.678473 3840 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-2rblw 7e1dad10171f9ac631e248b5027c5b39e2cb48d710e50335b181de870861e44c network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-2rblw 7e1dad10171f9ac631e248b5027c5b39e2cb48d710e50335b181de870861e44c network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:14:40:a5:87:71\",\"name\":\"7e1dad10171f9ac\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5a5fee64-bc2d-476b-bb54-ed76a6ba7eef\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:16:22.024674 3840 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:16:22.024695 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:16:22.024699 3840 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:16:22.024704 3840 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0326 23:16:22.077718 3840 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0326 23:16:22.115885 3840 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0326 23:16:22.429044 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:22.429069 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:22.429077 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0326 23:16:23.013775 3840 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:16:23.013796 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:16:23.013801 3840 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:16:23.013805 3840 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0326 23:16:23.373697 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0326 23:16:23.960876 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0326 23:16:23.986019 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:16:24.073009 3840 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:16:24.073026 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:16:24.466675 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:16:25.025818 3840 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0326 23:16:25.025835 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0326 23:16:29.486207 3840 base_network_controller_pods.go:487] [default/openshift-console/console-6bdc45c6d9-xhc8n] creating logical port openshift-console_console-6bdc45c6d9-xhc8n for pod on switch ip-10-0-2-103.ec2.internal I0326 23:16:29.486321 3840 kube.go:256] Updating pod openshift-console/console-6bdc45c6d9-xhc8n I0326 23:16:29.504255 3840 pod.go:62] [openshift-console/console-6bdc45c6d9-xhc8n] pod update took 17.975827ms I0326 23:16:29.504275 3840 base_network_controller_pods.go:937] [default/openshift-console/console-6bdc45c6d9-xhc8n] addLogicalPort annotation time took 18.001746ms I0326 23:16:29.505557 3840 pods.go:271] [openshift-console/console-6bdc45c6d9-xhc8n] addLogicalPort took 19.364831ms, libovsdb time 919.445µs W0326 23:16:33.054450 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0326 23:16:34.215277 3840 cni.go:403] [openshift-console/console-6bdc45c6d9-xhc8n c6b900fd3ea9c74dad84505b5a0a8da0e1883549dc4e55708893b49e4855e1cc network default NAD default] ADD starting CNI request [openshift-console/console-6bdc45c6d9-xhc8n c6b900fd3ea9c74dad84505b5a0a8da0e1883549dc4e55708893b49e4855e1cc network default NAD default] I0326 23:16:34.223254 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6bdc45c6d9-xhc8n, hostIfaceName: c6b900fd3ea9c74, network: default, NAD default, SandboxID: "c6b900fd3ea9c74dad84505b5a0a8da0e1883549dc4e55708893b49e4855e1cc", PCI device ID: , UID: "6ced0e61-7852-4617-91d0-153f84c785e3", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0326 23:16:34.265684 3840 cni.go:424] [openshift-console/console-6bdc45c6d9-xhc8n c6b900fd3ea9c74dad84505b5a0a8da0e1883549dc4e55708893b49e4855e1cc network default NAD default] ADD finished CNI request [openshift-console/console-6bdc45c6d9-xhc8n c6b900fd3ea9c74dad84505b5a0a8da0e1883549dc4e55708893b49e4855e1cc network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:5d:8d:76:ce:29\",\"name\":\"c6b900fd3ea9c74\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ee3bed8b-eb8d-42af-bfdd-745add14f85f\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:16:34.506130 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0326 23:16:35.006327 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0326 23:16:35.037175 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0326 23:16:35.037202 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0326 23:16:35.533050 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0326 23:16:36.132658 3840 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0326 23:16:37.520064 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:16:39.067727 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0326 23:16:40.093992 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:16:40.094023 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:16:44.965712 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:16:44.965738 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0326 23:16:44.980682 3840 cni.go:403] [openshift-multus/network-metrics-daemon-mqbfm 8054459883c9ba23e42134345afc3ccf028896be9f302d8724f0487e01a38748 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-mqbfm 8054459883c9ba23e42134345afc3ccf028896be9f302d8724f0487e01a38748 network default NAD default] I0326 23:16:44.989381 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-mqbfm, hostIfaceName: 8054459883c9ba2, network: default, NAD default, SandboxID: "8054459883c9ba23e42134345afc3ccf028896be9f302d8724f0487e01a38748", PCI device ID: , UID: "bc725b9f-36d3-4969-bf0b-78d2706223e2", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0326 23:16:45.239359 3840 cni.go:424] [openshift-multus/network-metrics-daemon-mqbfm 8054459883c9ba23e42134345afc3ccf028896be9f302d8724f0487e01a38748 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-mqbfm 8054459883c9ba23e42134345afc3ccf028896be9f302d8724f0487e01a38748 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:0d:ab:b1:41:c4\",\"name\":\"8054459883c9ba2\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ecff6309-6367-4d05-a2c2-767c2fc32277\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:16:45.477651 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0326 23:16:47.590721 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0326 23:16:47.590837 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0326 23:16:47.590899 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0326 23:16:48.442186 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:48.442206 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:48.442213 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:48.442233 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:48.442238 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:48.442243 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:48.478353 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:48.478371 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:48.478377 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:16:48.569607 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0326 23:16:48.727256 3840 obj_retry.go:464] Detected object openshift-image-registry/image-registry-56599b6895-b246v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:16:48.727781 3840 obj_retry.go:464] Detected object openshift-image-registry/image-registry-56599b6895-b246v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:16:49.113505 3840 obj_retry.go:458] Detected object openshift-image-registry/image-registry-56599b6895-b246v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:49.113526 3840 obj_retry.go:458] Detected object openshift-image-registry/image-registry-56599b6895-b246v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:16:49.118815 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-56599b6895-b246v I0326 23:16:49.118840 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-56599b6895-b246v W0326 23:16:50.080309 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:16:50.080329 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:16:50.126863 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:16:50.126882 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0326 23:17:03.127905 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0326 23:17:03.171913 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0326 23:17:04.680992 3840 cni.go:403] [openshift-console/console-6bdc45c6d9-xhc8n c6b900fd3ea9c74dad84505b5a0a8da0e1883549dc4e55708893b49e4855e1cc network default NAD default] DEL starting CNI request [openshift-console/console-6bdc45c6d9-xhc8n c6b900fd3ea9c74dad84505b5a0a8da0e1883549dc4e55708893b49e4855e1cc network default NAD default] I0326 23:17:04.763378 3840 cni.go:424] [openshift-console/console-6bdc45c6d9-xhc8n c6b900fd3ea9c74dad84505b5a0a8da0e1883549dc4e55708893b49e4855e1cc network default NAD default] DEL finished CNI request [openshift-console/console-6bdc45c6d9-xhc8n c6b900fd3ea9c74dad84505b5a0a8da0e1883549dc4e55708893b49e4855e1cc network default NAD default], result "{}", err I0326 23:17:04.836811 3840 obj_retry.go:464] Detected object openshift-console/console-6bdc45c6d9-xhc8n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:04.836842 3840 pods.go:176] Deleting pod: openshift-console/console-6bdc45c6d9-xhc8n I0326 23:17:04.838197 3840 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6bdc45c6d9-xhc8n, ips: 10.128.0.21 I0326 23:17:04.838228 3840 obj_retry.go:464] Detected object openshift-console/console-6bdc45c6d9-xhc8n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:04.844034 3840 base_network_controller_pods.go:487] [default/openshift-console/console-78d8d49547-w8r67] creating logical port openshift-console_console-78d8d49547-w8r67 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:17:04.844143 3840 kube.go:256] Updating pod openshift-console/console-78d8d49547-w8r67 I0326 23:17:04.860605 3840 pod.go:62] [openshift-console/console-78d8d49547-w8r67] pod update took 16.486081ms I0326 23:17:04.860624 3840 base_network_controller_pods.go:937] [default/openshift-console/console-78d8d49547-w8r67] addLogicalPort annotation time took 16.508817ms I0326 23:17:04.861729 3840 pods.go:271] [openshift-console/console-78d8d49547-w8r67] addLogicalPort took 17.708023ms, libovsdb time 803.263µs I0326 23:17:05.261872 3840 cni.go:403] [openshift-console/console-78d8d49547-w8r67 148e19a01e477bdaecff9f12d7d0166d70fa117b291d5a1d404aae04d7454afb network default NAD default] ADD starting CNI request [openshift-console/console-78d8d49547-w8r67 148e19a01e477bdaecff9f12d7d0166d70fa117b291d5a1d404aae04d7454afb network default NAD default] I0326 23:17:05.270489 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-78d8d49547-w8r67, hostIfaceName: 148e19a01e477bd, network: default, NAD default, SandboxID: "148e19a01e477bdaecff9f12d7d0166d70fa117b291d5a1d404aae04d7454afb", PCI device ID: , UID: "876cb3af-e501-446f-b178-c0ac47044309", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0326 23:17:05.314393 3840 cni.go:424] [openshift-console/console-78d8d49547-w8r67 148e19a01e477bdaecff9f12d7d0166d70fa117b291d5a1d404aae04d7454afb network default NAD default] ADD finished CNI request [openshift-console/console-78d8d49547-w8r67 148e19a01e477bdaecff9f12d7d0166d70fa117b291d5a1d404aae04d7454afb network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:93:47:01:70:e3\",\"name\":\"148e19a01e477bd\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/47b5196d-4a82-41e6-bd42-b45650301c00\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:17:05.649000 3840 obj_retry.go:458] Detected object openshift-console/console-6bdc45c6d9-xhc8n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:05.649031 3840 obj_retry.go:458] Detected object openshift-console/console-6bdc45c6d9-xhc8n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:05.653472 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6bdc45c6d9-xhc8n I0326 23:17:05.653497 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6bdc45c6d9-xhc8n I0326 23:17:15.403601 3840 obj_retry.go:464] Detected object openshift-console/console-6bdc45c6d9-pj9xj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:15.404142 3840 obj_retry.go:464] Detected object openshift-console/console-6bdc45c6d9-pj9xj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:17:15.685505 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:17:16.234515 3840 obj_retry.go:458] Detected object openshift-console/console-6bdc45c6d9-pj9xj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:16.234547 3840 obj_retry.go:458] Detected object openshift-console/console-6bdc45c6d9-pj9xj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:16.241863 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6bdc45c6d9-pj9xj I0326 23:17:16.241892 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6bdc45c6d9-pj9xj I0326 23:17:22.220643 3840 cni.go:403] [openshift-ingress/router-default-6565674565-gspwq 6f5d2bc6ad8ce1d4a4459598408cc5dc6f739a48929fde9848a069651628cac4 network default NAD default] DEL starting CNI request [openshift-ingress/router-default-6565674565-gspwq 6f5d2bc6ad8ce1d4a4459598408cc5dc6f739a48929fde9848a069651628cac4 network default NAD default] I0326 23:17:22.292296 3840 cni.go:424] [openshift-ingress/router-default-6565674565-gspwq 6f5d2bc6ad8ce1d4a4459598408cc5dc6f739a48929fde9848a069651628cac4 network default NAD default] DEL finished CNI request [openshift-ingress/router-default-6565674565-gspwq 6f5d2bc6ad8ce1d4a4459598408cc5dc6f739a48929fde9848a069651628cac4 network default NAD default], result "{}", err I0326 23:17:22.396468 3840 obj_retry.go:464] Detected object openshift-ingress/router-default-6565674565-gspwq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:22.396522 3840 pods.go:176] Deleting pod: openshift-ingress/router-default-6565674565-gspwq I0326 23:17:22.397799 3840 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-6565674565-gspwq, ips: 10.128.0.12 I0326 23:17:22.397828 3840 obj_retry.go:464] Detected object openshift-ingress/router-default-6565674565-gspwq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:22.706195 3840 obj_retry.go:458] Detected object openshift-ingress/router-default-6565674565-gspwq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:22.706217 3840 obj_retry.go:458] Detected object openshift-ingress/router-default-6565674565-gspwq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:22.711814 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-6565674565-gspwq I0326 23:17:22.711840 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-6565674565-gspwq I0326 23:17:38.309807 3840 cni.go:403] [openshift-monitoring/alertmanager-main-1 45ff77f32a0b402a2a21c7e07257b80d7f3a710f71f1e8cf4b05f7c1d86a9e0b network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 45ff77f32a0b402a2a21c7e07257b80d7f3a710f71f1e8cf4b05f7c1d86a9e0b network default NAD default] I0326 23:17:38.383245 3840 cni.go:424] [openshift-monitoring/alertmanager-main-1 45ff77f32a0b402a2a21c7e07257b80d7f3a710f71f1e8cf4b05f7c1d86a9e0b network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 45ff77f32a0b402a2a21c7e07257b80d7f3a710f71f1e8cf4b05f7c1d86a9e0b network default NAD default], result "{}", err I0326 23:17:38.443940 3840 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:38.443976 3840 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0326 23:17:38.445132 3840 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.128.0.17 I0326 23:17:38.445160 3840 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:38.777194 3840 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:38.777221 3840 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:38.782676 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0326 23:17:38.782705 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0326 23:17:38.823518 3840 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:17:38.823634 3840 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0326 23:17:38.847939 3840 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 24.36181ms I0326 23:17:38.847962 3840 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 24.389477ms I0326 23:17:38.849152 3840 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 25.646387ms, libovsdb time 881.514µs I0326 23:17:39.180550 3840 cni.go:403] [openshift-monitoring/alertmanager-main-1 45026e866607d30acef032de22b3397004845f1a233c50d9e7143e2dd233f094 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 45026e866607d30acef032de22b3397004845f1a233c50d9e7143e2dd233f094 network default NAD default] I0326 23:17:39.188683 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 45026e866607d30, network: default, NAD default, SandboxID: "45026e866607d30acef032de22b3397004845f1a233c50d9e7143e2dd233f094", PCI device ID: , UID: "1b259a83-2cab-44b7-bbe5-9158ba739aea", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0326 23:17:39.231166 3840 cni.go:424] [openshift-monitoring/alertmanager-main-1 45026e866607d30acef032de22b3397004845f1a233c50d9e7143e2dd233f094 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 45026e866607d30acef032de22b3397004845f1a233c50d9e7143e2dd233f094 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:bf:3e:7c:05:27\",\"name\":\"45026e866607d30\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e8d01e5a-3dc9-4039-b704-f91c76b47726\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:17:41.210554 3840 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-cf4558c45-9pr2n] creating logical port openshift-monitoring_telemeter-client-cf4558c45-9pr2n for pod on switch ip-10-0-2-103.ec2.internal I0326 23:17:41.210704 3840 kube.go:256] Updating pod openshift-monitoring/telemeter-client-cf4558c45-9pr2n I0326 23:17:41.234479 3840 pod.go:62] [openshift-monitoring/telemeter-client-cf4558c45-9pr2n] pod update took 23.807083ms I0326 23:17:41.234499 3840 base_network_controller_pods.go:937] [default/openshift-monitoring/telemeter-client-cf4558c45-9pr2n] addLogicalPort annotation time took 23.831945ms I0326 23:17:41.235694 3840 pods.go:271] [openshift-monitoring/telemeter-client-cf4558c45-9pr2n] addLogicalPort took 25.160904ms, libovsdb time 858.145µs I0326 23:17:41.569019 3840 cni.go:403] [openshift-monitoring/telemeter-client-cf4558c45-9pr2n 67c0a2e132cdabf9f1a6c3824ff6881d3e44c5e31e6482a6814317351969d4a1 network default NAD default] ADD starting CNI request [openshift-monitoring/telemeter-client-cf4558c45-9pr2n 67c0a2e132cdabf9f1a6c3824ff6881d3e44c5e31e6482a6814317351969d4a1 network default NAD default] I0326 23:17:41.577249 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-cf4558c45-9pr2n, hostIfaceName: 67c0a2e132cdabf, network: default, NAD default, SandboxID: "67c0a2e132cdabf9f1a6c3824ff6881d3e44c5e31e6482a6814317351969d4a1", PCI device ID: , UID: "d45bf26a-4ec6-42ea-8312-1cc1b501c440", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0326 23:17:41.617715 3840 cni.go:424] [openshift-monitoring/telemeter-client-cf4558c45-9pr2n 67c0a2e132cdabf9f1a6c3824ff6881d3e44c5e31e6482a6814317351969d4a1 network default NAD default] ADD finished CNI request [openshift-monitoring/telemeter-client-cf4558c45-9pr2n 67c0a2e132cdabf9f1a6c3824ff6881d3e44c5e31e6482a6814317351969d4a1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:a5:5f:40:6c:dd\",\"name\":\"67c0a2e132cdabf\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/30f4c30f-7182-4f96-b13e-2dfe92530a51\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:17:42.807764 3840 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:42.808372 3840 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:17:43.336262 3840 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:43.336291 3840 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:17:43.343437 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0326 23:17:43.343465 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 W0326 23:17:59.239066 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:17:59.239089 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0326 23:17:59.239095 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0326 23:18:00.551825 3840 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:00.552323 3840 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:01.364707 3840 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:01.364731 3840 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:01.372785 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0326 23:18:01.372804 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0326 23:18:10.230665 3840 cni.go:403] [openshift-console/console-78d8d49547-w8r67 148e19a01e477bdaecff9f12d7d0166d70fa117b291d5a1d404aae04d7454afb network default NAD default] DEL starting CNI request [openshift-console/console-78d8d49547-w8r67 148e19a01e477bdaecff9f12d7d0166d70fa117b291d5a1d404aae04d7454afb network default NAD default] I0326 23:18:10.297415 3840 cni.go:424] [openshift-console/console-78d8d49547-w8r67 148e19a01e477bdaecff9f12d7d0166d70fa117b291d5a1d404aae04d7454afb network default NAD default] DEL finished CNI request [openshift-console/console-78d8d49547-w8r67 148e19a01e477bdaecff9f12d7d0166d70fa117b291d5a1d404aae04d7454afb network default NAD default], result "{}", err I0326 23:18:10.344623 3840 obj_retry.go:464] Detected object openshift-console/console-78d8d49547-w8r67 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:10.344659 3840 pods.go:176] Deleting pod: openshift-console/console-78d8d49547-w8r67 I0326 23:18:10.346170 3840 pods.go:236] Attempting to release IPs for pod: openshift-console/console-78d8d49547-w8r67, ips: 10.128.0.22 I0326 23:18:10.346202 3840 obj_retry.go:464] Detected object openshift-console/console-78d8d49547-w8r67 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:10.356091 3840 base_network_controller_pods.go:487] [default/openshift-console/console-d864cb9c-bnzdr] creating logical port openshift-console_console-d864cb9c-bnzdr for pod on switch ip-10-0-2-103.ec2.internal I0326 23:18:10.356216 3840 kube.go:256] Updating pod openshift-console/console-d864cb9c-bnzdr I0326 23:18:10.371645 3840 pod.go:62] [openshift-console/console-d864cb9c-bnzdr] pod update took 15.452443ms I0326 23:18:10.371671 3840 base_network_controller_pods.go:937] [default/openshift-console/console-d864cb9c-bnzdr] addLogicalPort annotation time took 15.4824ms I0326 23:18:10.372844 3840 pods.go:271] [openshift-console/console-d864cb9c-bnzdr] addLogicalPort took 16.760666ms, libovsdb time 844.29µs I0326 23:18:10.714812 3840 cni.go:403] [openshift-console/console-d864cb9c-bnzdr 4935c84ac8be367966d8397e8ca6cba16f660786508d2cc2d95a302246a94553 network default NAD default] ADD starting CNI request [openshift-console/console-d864cb9c-bnzdr 4935c84ac8be367966d8397e8ca6cba16f660786508d2cc2d95a302246a94553 network default NAD default] I0326 23:18:10.724359 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-d864cb9c-bnzdr, hostIfaceName: 4935c84ac8be367, network: default, NAD default, SandboxID: "4935c84ac8be367966d8397e8ca6cba16f660786508d2cc2d95a302246a94553", PCI device ID: , UID: "54d99823-abbb-4b81-bed4-f27deca34237", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0326 23:18:10.765970 3840 cni.go:424] [openshift-console/console-d864cb9c-bnzdr 4935c84ac8be367966d8397e8ca6cba16f660786508d2cc2d95a302246a94553 network default NAD default] ADD finished CNI request [openshift-console/console-d864cb9c-bnzdr 4935c84ac8be367966d8397e8ca6cba16f660786508d2cc2d95a302246a94553 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:fe:43:ec:9e:5e\",\"name\":\"4935c84ac8be367\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d7377cf4-319f-43e1-bf08-d9cc409e96dc\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:18:10.944772 3840 obj_retry.go:458] Detected object openshift-console/console-78d8d49547-w8r67 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:10.944827 3840 obj_retry.go:458] Detected object openshift-console/console-78d8d49547-w8r67 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:10.947065 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-78d8d49547-w8r67 I0326 23:18:10.947096 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-78d8d49547-w8r67 W0326 23:18:20.977759 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:18:21.739042 3840 obj_retry.go:464] Detected object openshift-console/console-78d8d49547-mqwlr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:21.739752 3840 obj_retry.go:464] Detected object openshift-console/console-78d8d49547-mqwlr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:22.450387 3840 obj_retry.go:458] Detected object openshift-console/console-78d8d49547-mqwlr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:22.450409 3840 obj_retry.go:458] Detected object openshift-console/console-78d8d49547-mqwlr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:22.458609 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-78d8d49547-mqwlr I0326 23:18:22.458629 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-78d8d49547-mqwlr I0326 23:18:25.434950 3840 egressfirewall.go:628] Syncing node "ip-10-0-1-99.ec2.internal" for egress firewall I0326 23:18:25.434976 3840 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-99.ec2.internal I0326 23:18:25.435018 3840 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-99.ec2.internal: 52.774µs I0326 23:18:30.647941 3840 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0326 23:18:45.955449 3840 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:45.956073 3840 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:18:46.563317 3840 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:46.563340 3840 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:18:46.572887 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0326 23:18:46.572913 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0326 23:19:59.619131 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=5 I0326 23:20:00.192770 3840 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29576120-x4f4x for pod on switch ip-10-0-2-103.ec2.internal I0326 23:20:00.192898 3840 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x I0326 23:20:00.208339 3840 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x] pod update took 15.467387ms I0326 23:20:00.208362 3840 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x] addLogicalPort annotation time took 15.495976ms I0326 23:20:00.209504 3840 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x] addLogicalPort took 16.749006ms, libovsdb time 825.741µs I0326 23:20:00.572188 3840 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x 2cff1da6a901fe883d93780fb25781b0cbcceb5ac8dc5a784057d71ffaec8377 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x 2cff1da6a901fe883d93780fb25781b0cbcceb5ac8dc5a784057d71ffaec8377 network default NAD default] I0326 23:20:00.580441 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29576120-x4f4x, hostIfaceName: 2cff1da6a901fe8, network: default, NAD default, SandboxID: "2cff1da6a901fe883d93780fb25781b0cbcceb5ac8dc5a784057d71ffaec8377", PCI device ID: , UID: "59d66ad6-6b98-4f2d-8940-fcf9ef08139a", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0326 23:20:00.622657 3840 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x 2cff1da6a901fe883d93780fb25781b0cbcceb5ac8dc5a784057d71ffaec8377 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x 2cff1da6a901fe883d93780fb25781b0cbcceb5ac8dc5a784057d71ffaec8377 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:a0:f6:f5:08:58\",\"name\":\"2cff1da6a901fe8\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/83c54e78-d700-416e-a5f7-c7d62fcad139\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:20:05.377993 3840 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x 2cff1da6a901fe883d93780fb25781b0cbcceb5ac8dc5a784057d71ffaec8377 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x 2cff1da6a901fe883d93780fb25781b0cbcceb5ac8dc5a784057d71ffaec8377 network default NAD default] I0326 23:20:05.452545 3840 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x 2cff1da6a901fe883d93780fb25781b0cbcceb5ac8dc5a784057d71ffaec8377 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x 2cff1da6a901fe883d93780fb25781b0cbcceb5ac8dc5a784057d71ffaec8377 network default NAD default], result "{}", err I0326 23:20:05.494065 3840 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:05.494104 3840 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x I0326 23:20:05.495270 3840 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x, ips: 10.128.0.26 I0326 23:20:05.495297 3840 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:06.514089 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:06.514119 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:26.543693 3840 cni.go:403] [openshift-console/console-d864cb9c-bnzdr 4935c84ac8be367966d8397e8ca6cba16f660786508d2cc2d95a302246a94553 network default NAD default] DEL starting CNI request [openshift-console/console-d864cb9c-bnzdr 4935c84ac8be367966d8397e8ca6cba16f660786508d2cc2d95a302246a94553 network default NAD default] I0326 23:20:26.618668 3840 cni.go:424] [openshift-console/console-d864cb9c-bnzdr 4935c84ac8be367966d8397e8ca6cba16f660786508d2cc2d95a302246a94553 network default NAD default] DEL finished CNI request [openshift-console/console-d864cb9c-bnzdr 4935c84ac8be367966d8397e8ca6cba16f660786508d2cc2d95a302246a94553 network default NAD default], result "{}", err I0326 23:20:26.667973 3840 obj_retry.go:464] Detected object openshift-console/console-d864cb9c-bnzdr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:26.668021 3840 pods.go:176] Deleting pod: openshift-console/console-d864cb9c-bnzdr I0326 23:20:26.669283 3840 pods.go:236] Attempting to release IPs for pod: openshift-console/console-d864cb9c-bnzdr, ips: 10.128.0.25 I0326 23:20:26.669312 3840 obj_retry.go:464] Detected object openshift-console/console-d864cb9c-bnzdr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:26.673734 3840 base_network_controller_pods.go:487] [default/openshift-console/console-84dc4cf488-dkw5z] creating logical port openshift-console_console-84dc4cf488-dkw5z for pod on switch ip-10-0-2-103.ec2.internal I0326 23:20:26.673841 3840 kube.go:256] Updating pod openshift-console/console-84dc4cf488-dkw5z I0326 23:20:26.690132 3840 pod.go:62] [openshift-console/console-84dc4cf488-dkw5z] pod update took 16.323803ms I0326 23:20:26.690154 3840 base_network_controller_pods.go:937] [default/openshift-console/console-84dc4cf488-dkw5z] addLogicalPort annotation time took 16.35085ms I0326 23:20:26.691263 3840 pods.go:271] [openshift-console/console-84dc4cf488-dkw5z] addLogicalPort took 17.541489ms, libovsdb time 780.683µs I0326 23:20:27.032224 3840 cni.go:403] [openshift-console/console-84dc4cf488-dkw5z 9987f23e2ebf013867851a9aa6b0a14f725688f415686e4063164704a2aead91 network default NAD default] ADD starting CNI request [openshift-console/console-84dc4cf488-dkw5z 9987f23e2ebf013867851a9aa6b0a14f725688f415686e4063164704a2aead91 network default NAD default] I0326 23:20:27.040534 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-84dc4cf488-dkw5z, hostIfaceName: 9987f23e2ebf013, network: default, NAD default, SandboxID: "9987f23e2ebf013867851a9aa6b0a14f725688f415686e4063164704a2aead91", PCI device ID: , UID: "737b7101-e174-4f01-a061-cd8149c7b267", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0326 23:20:27.082119 3840 cni.go:424] [openshift-console/console-84dc4cf488-dkw5z 9987f23e2ebf013867851a9aa6b0a14f725688f415686e4063164704a2aead91 network default NAD default] ADD finished CNI request [openshift-console/console-84dc4cf488-dkw5z 9987f23e2ebf013867851a9aa6b0a14f725688f415686e4063164704a2aead91 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:87:d2:f9:9c:fb\",\"name\":\"9987f23e2ebf013\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1fe47976-4e9c-4646-b28f-813ed2438ef0\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:20:27.477006 3840 obj_retry.go:458] Detected object openshift-console/console-d864cb9c-bnzdr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:27.477031 3840 obj_retry.go:458] Detected object openshift-console/console-d864cb9c-bnzdr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:27.482644 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-d864cb9c-bnzdr I0326 23:20:27.482671 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-d864cb9c-bnzdr I0326 23:20:36.645308 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0326 23:20:37.248861 3840 obj_retry.go:464] Detected object openshift-console/console-d864cb9c-9mtj7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:20:37.249530 3840 obj_retry.go:464] Detected object openshift-console/console-d864cb9c-9mtj7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:20:37.508111 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0326 23:20:38.023852 3840 obj_retry.go:458] Detected object openshift-console/console-d864cb9c-9mtj7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:38.023880 3840 obj_retry.go:458] Detected object openshift-console/console-d864cb9c-9mtj7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:20:38.026852 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-d864cb9c-9mtj7 I0326 23:20:38.026887 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-d864cb9c-9mtj7 I0326 23:20:47.537863 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0326 23:20:50.544967 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=70 I0326 23:20:53.619977 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0326 23:21:06.640336 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0326 23:21:17.656286 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0326 23:21:23.432378 3840 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-fk77j] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-fk77j for pod on switch ip-10-0-2-103.ec2.internal I0326 23:21:23.432525 3840 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-fk77j I0326 23:21:23.449402 3840 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-fk77j] pod update took 16.912753ms I0326 23:21:23.449424 3840 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-fk77j] addLogicalPort annotation time took 16.939558ms I0326 23:21:23.450652 3840 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-fk77j] addLogicalPort took 18.290614ms, libovsdb time 880.47µs I0326 23:21:23.790818 3840 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-fk77j c32452887c4d6ed0a0c2328f945002a9d73dfac814ec46dce99faa5420063fce network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-fk77j c32452887c4d6ed0a0c2328f945002a9d73dfac814ec46dce99faa5420063fce network default NAD default] I0326 23:21:23.800360 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-fk77j, hostIfaceName: c32452887c4d6ed, network: default, NAD default, SandboxID: "c32452887c4d6ed0a0c2328f945002a9d73dfac814ec46dce99faa5420063fce", PCI device ID: , UID: "3f482b8c-5b75-4d85-b2db-bbef5b7074db", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0326 23:21:24.051444 3840 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-fk77j c32452887c4d6ed0a0c2328f945002a9d73dfac814ec46dce99faa5420063fce network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-fk77j c32452887c4d6ed0a0c2328f945002a9d73dfac814ec46dce99faa5420063fce network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:88:9e:dd:e2:4e\",\"name\":\"c32452887c4d6ed\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7f353755-2e6a-4b12-ae6d-05d5431e57d7\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:21:26.696028 3840 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:21:32.545435 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=147 W0326 23:21:37.709737 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0326 23:21:37.740348 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0326 23:21:37.740417 3840 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-x7gvg IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-x7gvg: no pod IPs found I0326 23:21:44.544159 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0326 23:22:00.624412 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0326 23:23:28.630672 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0326 23:23:53.539089 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=72 I0326 23:23:54.550086 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=663 I0326 23:24:01.656019 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0326 23:24:14.650441 3840 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0326 23:24:32.634785 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0326 23:26:01.105619 3840 namespace.go:98] [tsf] adding namespace I0326 23:26:01.106568 3840 namespace.go:102] [tsf] adding namespace took 923.192µs I0326 23:26:01.126969 3840 namespace.go:142] [tsf] updating namespace I0326 23:26:01.137156 3840 namespace.go:142] [tsf] updating namespace I0326 23:26:04.779446 3840 namespace.go:98] [konflux-ui] adding namespace I0326 23:26:04.780334 3840 namespace.go:102] [konflux-ui] adding namespace took 865.296µs I0326 23:26:04.789221 3840 namespace.go:98] [rhtpa-operator] adding namespace I0326 23:26:04.789329 3840 namespace.go:98] [konflux-operator] adding namespace I0326 23:26:04.789453 3840 namespace.go:98] [cert-manager-operator] adding namespace I0326 23:26:04.789562 3840 namespace.go:98] [tsf-tas] adding namespace I0326 23:26:04.789818 3840 namespace.go:98] [rhbk-operator] adding namespace I0326 23:26:04.789895 3840 namespace.go:98] [tsf-keycloak] adding namespace I0326 23:26:04.790072 3840 namespace.go:102] [rhtpa-operator] adding namespace took 827.084µs I0326 23:26:04.790691 3840 namespace.go:102] [konflux-operator] adding namespace took 1.344931ms I0326 23:26:04.791229 3840 namespace.go:102] [cert-manager-operator] adding namespace took 1.750589ms I0326 23:26:04.791750 3840 namespace.go:102] [tsf-tas] adding namespace took 2.176159ms I0326 23:26:04.791779 3840 namespace.go:98] [tsf-tpa] adding namespace I0326 23:26:04.792259 3840 namespace.go:102] [tsf-tpa] adding namespace took 473.986µs I0326 23:26:04.792811 3840 namespace.go:102] [rhbk-operator] adding namespace took 2.975551ms I0326 23:26:04.793204 3840 namespace.go:102] [tsf-keycloak] adding namespace took 3.294625ms I0326 23:26:04.800642 3840 namespace.go:142] [konflux-ui] updating namespace I0326 23:26:04.813906 3840 namespace.go:142] [konflux-ui] updating namespace I0326 23:26:04.824638 3840 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:26:04.827102 3840 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:26:04.830985 3840 namespace.go:142] [konflux-operator] updating namespace I0326 23:26:04.831011 3840 namespace.go:142] [tsf-tas] updating namespace I0326 23:26:04.839959 3840 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:04.840039 3840 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:04.842420 3840 namespace.go:142] [rhbk-operator] updating namespace I0326 23:26:04.842565 3840 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:26:04.849844 3840 namespace.go:142] [konflux-ui] updating namespace I0326 23:26:04.857553 3840 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:26:04.877982 3840 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:26:04.891666 3840 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:26:04.899918 3840 namespace.go:142] [tsf-tas] updating namespace I0326 23:26:04.913218 3840 namespace.go:142] [tsf-tas] updating namespace I0326 23:26:04.921220 3840 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:04.933935 3840 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:04.944144 3840 namespace.go:142] [rhbk-operator] updating namespace I0326 23:26:04.961668 3840 namespace.go:142] [rhbk-operator] updating namespace I0326 23:26:04.967022 3840 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:04.978683 3840 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:04.982432 3840 namespace.go:142] [konflux-operator] updating namespace I0326 23:26:04.997605 3840 namespace.go:142] [konflux-operator] updating namespace I0326 23:26:05.829067 3840 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:05.829944 3840 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:11.475726 3840 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8bcmxc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:11.476295 3840 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8bcmxc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:11.476320 3840 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8bcmxc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:11.476329 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:11.476339 3840 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8bcmxc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:11.476345 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:11.511642 3840 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfz748x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:11.512051 3840 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfz748x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:11.512072 3840 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfz748x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:11.512079 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:11.512089 3840 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfz748x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:11.512093 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:12.501356 3840 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8bcmxc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:12.501375 3840 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8bcmxc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:12.501385 3840 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8bcmxc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:12.501396 3840 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8bcmxc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:12.538715 3840 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfz748x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:12.538733 3840 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfz748x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:12.538746 3840 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfz748x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:12.538757 3840 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfz748x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:16.544105 3840 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1lshpz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:16.544797 3840 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1lshpz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:16.544839 3840 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1lshpz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:16.544855 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:16.544872 3840 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1lshpz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:16.544878 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:17.570429 3840 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1lshpz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:17.570458 3840 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1lshpz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:17.570487 3840 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1lshpz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:17.570499 3840 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1lshpz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:19.545347 3840 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafzphh9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:19.545933 3840 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafzphh9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:19.545968 3840 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafzphh9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:19.545979 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:19.545996 3840 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafzphh9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:19.546003 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:19.581865 3840 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7kpcfk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:19.582401 3840 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7kpcfk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:19.582430 3840 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7kpcfk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:19.582438 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:19.582450 3840 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7kpcfk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:19.582454 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:20.587411 3840 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafzphh9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:20.587440 3840 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafzphh9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:20.587463 3840 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafzphh9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:20.587485 3840 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafzphh9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:20.618780 3840 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7kpcfk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:20.618802 3840 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7kpcfk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:20.618817 3840 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7kpcfk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:20.618828 3840 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7kpcfk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:21.359146 3840 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-xvktb] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-xvktb for pod on switch ip-10-0-2-103.ec2.internal I0326 23:26:21.359258 3840 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-xvktb I0326 23:26:21.374979 3840 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-xvktb] pod update took 15.747588ms I0326 23:26:21.375000 3840 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-xvktb] addLogicalPort annotation time took 15.774168ms I0326 23:26:21.376229 3840 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-xvktb] addLogicalPort took 17.098864ms, libovsdb time 907.951µs I0326 23:26:21.715853 3840 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-xvktb 02cb63a41626fdd47f23064286cc4f84940f2279462c35b39bd759036aac254f network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-xvktb 02cb63a41626fdd47f23064286cc4f84940f2279462c35b39bd759036aac254f network default NAD default] I0326 23:26:21.724803 3840 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-xvktb, hostIfaceName: 02cb63a41626fdd, network: default, NAD default, SandboxID: "02cb63a41626fdd47f23064286cc4f84940f2279462c35b39bd759036aac254f", PCI device ID: , UID: "13834ac7-ff06-4499-a5cf-f583b4c38776", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0326 23:26:21.767366 3840 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-xvktb 02cb63a41626fdd47f23064286cc4f84940f2279462c35b39bd759036aac254f network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-xvktb 02cb63a41626fdd47f23064286cc4f84940f2279462c35b39bd759036aac254f network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:a2:7d:ba:e7:d6\",\"name\":\"02cb63a41626fdd\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d158b799-f88f-4d60-9152-f4581107ac89\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:26:23.643788 3840 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5hw6jz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:23.644376 3840 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5hw6jz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:26:23.644409 3840 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5hw6jz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:23.644419 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0326 23:26:23.644440 3840 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5hw6jz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0326 23:26:23.644448 3840 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0326 23:26:24.688176 3840 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5hw6jz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:24.688203 3840 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5hw6jz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:24.688213 3840 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5hw6jz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:26:24.688229 3840 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5hw6jz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:26:24.909872 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0326 23:26:25.623073 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0326 23:26:26.738555 3840 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-wgpdf for pod on switch ip-10-0-2-103.ec2.internal I0326 23:26:26.738777 3840 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf I0326 23:26:26.759234 3840 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf] pod update took 20.500926ms I0326 23:26:26.759255 3840 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf] addLogicalPort annotation time took 20.527796ms I0326 23:26:26.760551 3840 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf] addLogicalPort took 22.01343ms, libovsdb time 923.644µs I0326 23:26:27.096277 3840 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf 5ce72f8230fb64d9dc2b274a04c09f6fa510f6eb8c84b0e38318268a4a74b130 network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf 5ce72f8230fb64d9dc2b274a04c09f6fa510f6eb8c84b0e38318268a4a74b130 network default NAD default] I0326 23:26:27.104522 3840 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-wgpdf, hostIfaceName: 5ce72f8230fb64d, network: default, NAD default, SandboxID: "5ce72f8230fb64d9dc2b274a04c09f6fa510f6eb8c84b0e38318268a4a74b130", PCI device ID: , UID: "392b6465-5a3d-4a24-ab81-a0109a21d60d", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0326 23:26:27.146204 3840 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf 5ce72f8230fb64d9dc2b274a04c09f6fa510f6eb8c84b0e38318268a4a74b130 network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf 5ce72f8230fb64d9dc2b274a04c09f6fa510f6eb8c84b0e38318268a4a74b130 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:02:0e:4a:e4:2b\",\"name\":\"5ce72f8230fb64d\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9e007359-4a45-4586-af84-d17c41490014\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:26:29.929097 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:26:30.830431 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:26:30.848025 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:26:30.937067 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0326 23:26:30.955020 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0326 23:26:32.280018 3840 namespace.go:142] [openshift-operators] updating namespace I0326 23:26:32.291741 3840 namespace.go:142] [openshift-operators] updating namespace W0326 23:26:35.925163 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0326 23:26:39.229362 3840 namespace.go:142] [openshift-operators] updating namespace I0326 23:26:39.524938 3840 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-4vgpc] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-4vgpc for pod on switch ip-10-0-2-103.ec2.internal I0326 23:26:39.525085 3840 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-4vgpc I0326 23:26:39.546900 3840 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-dkm6f] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-dkm6f for pod on switch ip-10-0-2-103.ec2.internal I0326 23:26:39.547046 3840 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-dkm6f I0326 23:26:39.557716 3840 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-4vgpc] pod update took 32.658298ms I0326 23:26:39.557739 3840 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-4vgpc] addLogicalPort annotation time took 32.686975ms I0326 23:26:39.558863 3840 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-4vgpc] addLogicalPort took 33.942836ms, libovsdb time 818.053µs I0326 23:26:39.572849 3840 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-dkm6f] pod update took 25.829084ms I0326 23:26:39.572870 3840 base_network_controller_pods.go:937] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-dkm6f] addLogicalPort annotation time took 25.854664ms I0326 23:26:39.574025 3840 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-dkm6f] addLogicalPort took 27.139436ms, libovsdb time 859.326µs I0326 23:26:39.900910 3840 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-4vgpc f70bf5259d55300b5f8b28e461ad2e0d4d4fa58174ce3f45841eed7baf776ccf network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-4vgpc f70bf5259d55300b5f8b28e461ad2e0d4d4fa58174ce3f45841eed7baf776ccf network default NAD default] I0326 23:26:39.908350 3840 cni.go:403] [openshift-operators/tekton-operator-webhook-668b6fdd5d-dkm6f cf7b636c9076204dea740ad029277bc5ce6dba43c1a730da4e755b554b668251 network default NAD default] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-dkm6f cf7b636c9076204dea740ad029277bc5ce6dba43c1a730da4e755b554b668251 network default NAD default] I0326 23:26:39.909394 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-4vgpc, hostIfaceName: f70bf5259d55300, network: default, NAD default, SandboxID: "f70bf5259d55300b5f8b28e461ad2e0d4d4fa58174ce3f45841eed7baf776ccf", PCI device ID: , UID: "7d11c4b1-145b-4b21-b162-2650160a5533", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0326 23:26:39.916574 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-dkm6f, hostIfaceName: cf7b636c9076204, network: default, NAD default, SandboxID: "cf7b636c9076204dea740ad029277bc5ce6dba43c1a730da4e755b554b668251", PCI device ID: , UID: "0c98890b-1230-4563-8c44-1734507849da", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0326 23:26:39.955441 3840 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-4vgpc f70bf5259d55300b5f8b28e461ad2e0d4d4fa58174ce3f45841eed7baf776ccf network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-4vgpc f70bf5259d55300b5f8b28e461ad2e0d4d4fa58174ce3f45841eed7baf776ccf network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:1c:ab:0e:54:9b\",\"name\":\"f70bf5259d55300\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f067db3e-54d8-4df1-9cbb-ff4393e4f399\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:26:39.977201 3840 cni.go:424] [openshift-operators/tekton-operator-webhook-668b6fdd5d-dkm6f cf7b636c9076204dea740ad029277bc5ce6dba43c1a730da4e755b554b668251 network default NAD default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-dkm6f cf7b636c9076204dea740ad029277bc5ce6dba43c1a730da4e755b554b668251 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:a6:f5:e9:ca:89\",\"name\":\"cf7b636c9076204\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ff30bcc-4336-4584-b33b-4d6fd17e6d4a\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:26:40.973912 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0326 23:26:45.021286 3840 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8jpf2] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-8jpf2 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:26:45.021405 3840 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8jpf2 I0326 23:26:45.040006 3840 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8jpf2] pod update took 18.629831ms I0326 23:26:45.040032 3840 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8jpf2] addLogicalPort annotation time took 18.6539ms I0326 23:26:45.041150 3840 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8jpf2] addLogicalPort took 19.883153ms, libovsdb time 790.547µs I0326 23:26:45.684952 3840 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8jpf2 335ffa2ade674023ab3fc9d0d0bc052068bdf690b353ad2888b28d4c735106e4 network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8jpf2 335ffa2ade674023ab3fc9d0d0bc052068bdf690b353ad2888b28d4c735106e4 network default NAD default] I0326 23:26:45.693240 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-8jpf2, hostIfaceName: 335ffa2ade67402, network: default, NAD default, SandboxID: "335ffa2ade674023ab3fc9d0d0bc052068bdf690b353ad2888b28d4c735106e4", PCI device ID: , UID: "5f4608b1-124e-48eb-bea3-a80f2d593538", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0326 23:26:45.738021 3840 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8jpf2 335ffa2ade674023ab3fc9d0d0bc052068bdf690b353ad2888b28d4c735106e4 network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8jpf2 335ffa2ade674023ab3fc9d0d0bc052068bdf690b353ad2888b28d4c735106e4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:21:a0:50:2c:4f\",\"name\":\"335ffa2ade67402\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2ee029fd-0770-41a1-86da-45e269e3639d\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:26:47.989262 3840 namespace.go:98] [trusted-artifact-signer] adding namespace I0326 23:26:47.990229 3840 namespace.go:102] [trusted-artifact-signer] adding namespace took 948.425µs W0326 23:26:48.193814 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0326 23:26:48.197764 3840 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:26:48.234061 3840 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:26:51.404168 3840 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-87l7m] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-87l7m for pod on switch ip-10-0-2-103.ec2.internal I0326 23:26:51.404312 3840 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-87l7m I0326 23:26:51.440090 3840 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-87l7m] pod update took 35.806466ms I0326 23:26:51.440111 3840 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-87l7m] addLogicalPort annotation time took 35.831525ms I0326 23:26:51.441328 3840 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-87l7m] addLogicalPort took 37.175868ms, libovsdb time 857.414µs I0326 23:26:51.752625 3840 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-87l7m b6fee9ae0412b12520f075791f6798be31a1c5a610a3a8d25da926b875b7c2d9 network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-87l7m b6fee9ae0412b12520f075791f6798be31a1c5a610a3a8d25da926b875b7c2d9 network default NAD default] I0326 23:26:51.760826 3840 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-87l7m, hostIfaceName: b6fee9ae0412b12, network: default, NAD default, SandboxID: "b6fee9ae0412b12520f075791f6798be31a1c5a610a3a8d25da926b875b7c2d9", PCI device ID: , UID: "2fd2561b-18dc-4e62-b6f1-74d5ef4b88ef", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0326 23:26:51.805000 3840 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-87l7m b6fee9ae0412b12520f075791f6798be31a1c5a610a3a8d25da926b875b7c2d9 network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-87l7m b6fee9ae0412b12520f075791f6798be31a1c5a610a3a8d25da926b875b7c2d9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:d4:a2:9a:5d:7a\",\"name\":\"b6fee9ae0412b12\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0774aca6-aaeb-4aeb-b27c-e1ee4682b0bb\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:26:54.091002 3840 namespace.go:98] [openshift-pipelines] adding namespace I0326 23:26:54.091842 3840 namespace.go:102] [openshift-pipelines] adding namespace took 815.084µs I0326 23:26:54.117924 3840 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:26:54.921124 3840 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:26:55.498959 3840 namespace.go:142] [konflux-operator] updating namespace I0326 23:26:55.967102 3840 namespace.go:98] [cert-manager] adding namespace I0326 23:26:55.967936 3840 namespace.go:102] [cert-manager] adding namespace took 812.791µs I0326 23:26:56.194519 3840 namespace.go:142] [cert-manager] updating namespace I0326 23:26:56.259976 3840 namespace.go:142] [konflux-ui] updating namespace I0326 23:26:56.279766 3840 namespace.go:142] [cert-manager] updating namespace I0326 23:26:56.649678 3840 namespace.go:142] [open-cluster-management-2p9mp5cv8t0e1bpnh305hr8ld9shop2c] updating namespace I0326 23:26:57.104509 3840 namespace.go:142] [rhbk-operator] updating namespace I0326 23:26:57.299786 3840 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:26:57.499499 3840 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:26:57.693780 3840 namespace.go:142] [tsf] updating namespace I0326 23:26:57.898764 3840 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:58.106814 3840 namespace.go:142] [tsf-tas] updating namespace I0326 23:26:58.308279 3840 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:58.440681 3840 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:26:58.448163 3840 namespace.go:142] [default] updating namespace I0326 23:26:58.459933 3840 namespace.go:142] [konflux-operator] updating namespace I0326 23:26:58.469230 3840 namespace.go:142] [konflux-ui] updating namespace I0326 23:26:58.478936 3840 namespace.go:142] [open-cluster-management-2p9mp5cv8t0e1bpnh305hr8ld9shop2c] updating namespace I0326 23:26:58.488788 3840 namespace.go:142] [openshift] updating namespace I0326 23:26:58.499112 3840 namespace.go:142] [rhbk-operator] updating namespace I0326 23:26:58.512107 3840 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:26:58.523735 3840 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:26:58.535347 3840 namespace.go:142] [tsf] updating namespace I0326 23:26:58.545718 3840 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:58.557197 3840 namespace.go:142] [tsf-tas] updating namespace I0326 23:26:58.567400 3840 namespace.go:142] [tsf-tpa] updating namespace I0326 23:26:58.662713 3840 namespace.go:142] [cert-manager-operator] updating namespace I0326 23:26:58.763253 3840 namespace.go:142] [default] updating namespace I0326 23:26:58.868393 3840 namespace.go:142] [konflux-operator] updating namespace I0326 23:26:58.966083 3840 namespace.go:142] [konflux-ui] updating namespace W0326 23:26:59.052246 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0326 23:26:59.064623 3840 namespace.go:142] [open-cluster-management-2p9mp5cv8t0e1bpnh305hr8ld9shop2c] updating namespace I0326 23:26:59.161461 3840 namespace.go:142] [openshift] updating namespace I0326 23:26:59.261988 3840 namespace.go:142] [rhbk-operator] updating namespace I0326 23:26:59.372066 3840 namespace.go:142] [rhtpa-operator] updating namespace I0326 23:26:59.468153 3840 namespace.go:142] [trusted-artifact-signer] updating namespace I0326 23:26:59.565670 3840 namespace.go:142] [tsf] updating namespace I0326 23:26:59.662876 3840 namespace.go:142] [tsf-keycloak] updating namespace I0326 23:26:59.763342 3840 namespace.go:142] [tsf-tas] updating namespace I0326 23:26:59.863448 3840 namespace.go:142] [tsf-tpa] updating namespace I0326 23:27:00.502141 3840 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:27:00.757063 3840 namespace.go:142] [cert-manager] updating namespace I0326 23:27:00.886213 3840 namespace.go:142] [cert-manager] updating namespace I0326 23:27:00.983495 3840 namespace.go:142] [cert-manager] updating namespace W0326 23:27:05.637530 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0326 23:27:05.637580 3840 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0326 23:27:08.344904 3840 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-54t7t] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-54t7t for pod on switch ip-10-0-2-103.ec2.internal I0326 23:27:08.345021 3840 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-54t7t I0326 23:27:08.363521 3840 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-54t7t] pod update took 18.524022ms I0326 23:27:08.363542 3840 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-54t7t] addLogicalPort annotation time took 18.550309ms I0326 23:27:08.364877 3840 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-54t7t] addLogicalPort took 19.988223ms, libovsdb time 927.875µs I0326 23:27:08.533512 3840 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-n9j97] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-n9j97 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:27:08.533652 3840 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-n9j97 I0326 23:27:08.549515 3840 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-n9j97] pod update took 15.886724ms I0326 23:27:08.549532 3840 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-n9j97] addLogicalPort annotation time took 15.909028ms I0326 23:27:08.550780 3840 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-n9j97] addLogicalPort took 17.283372ms, libovsdb time 906.01µs I0326 23:27:08.642216 3840 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-7wk8k] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-7wk8k for pod on switch ip-10-0-2-103.ec2.internal I0326 23:27:08.642350 3840 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-7wk8k I0326 23:27:08.658684 3840 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-7wk8k] pod update took 16.365371ms I0326 23:27:08.658706 3840 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-7wk8k] addLogicalPort annotation time took 16.393283ms I0326 23:27:08.659916 3840 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-7wk8k] addLogicalPort took 17.719678ms, libovsdb time 837.232µs I0326 23:27:08.712954 3840 cni.go:403] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-54t7t 6f665ae1ac63be75e7c9410a08a25154228b4ba7240a785f82ee2ce85ddacccf network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-54t7t 6f665ae1ac63be75e7c9410a08a25154228b4ba7240a785f82ee2ce85ddacccf network default NAD default] I0326 23:27:08.722679 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-54t7t, hostIfaceName: 6f665ae1ac63be7, network: default, NAD default, SandboxID: "6f665ae1ac63be75e7c9410a08a25154228b4ba7240a785f82ee2ce85ddacccf", PCI device ID: , UID: "c81c8435-2aa7-450a-9982-32e85b1f23bc", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0326 23:27:08.736429 3840 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-rnshr] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-7988b548df-rnshr for pod on switch ip-10-0-2-103.ec2.internal I0326 23:27:08.736559 3840 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-rnshr I0326 23:27:08.750311 3840 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-rnshr] pod update took 13.795952ms I0326 23:27:08.750336 3840 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-rnshr] addLogicalPort annotation time took 13.828129ms I0326 23:27:08.751830 3840 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-rnshr] addLogicalPort took 15.423875ms, libovsdb time 1.014401ms I0326 23:27:08.768795 3840 cni.go:424] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-54t7t 6f665ae1ac63be75e7c9410a08a25154228b4ba7240a785f82ee2ce85ddacccf network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-54t7t 6f665ae1ac63be75e7c9410a08a25154228b4ba7240a785f82ee2ce85ddacccf network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:b7:b6:f1:b8:60\",\"name\":\"6f665ae1ac63be7\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/44d22e73-057a-459a-a8ed-2729866e4b26\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:27:08.893652 3840 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-n9j97 511df599ad05d379cc6663e550964c288a25d4037f617fb0a1015e4ecaee5156 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-n9j97 511df599ad05d379cc6663e550964c288a25d4037f617fb0a1015e4ecaee5156 network default NAD default] I0326 23:27:08.903534 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-n9j97, hostIfaceName: 511df599ad05d37, network: default, NAD default, SandboxID: "511df599ad05d379cc6663e550964c288a25d4037f617fb0a1015e4ecaee5156", PCI device ID: , UID: "5e98b117-757a-4035-afbe-589a9734e7d8", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0326 23:27:08.951230 3840 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-n9j97 511df599ad05d379cc6663e550964c288a25d4037f617fb0a1015e4ecaee5156 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-n9j97 511df599ad05d379cc6663e550964c288a25d4037f617fb0a1015e4ecaee5156 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:22:14:07:e9:56\",\"name\":\"511df599ad05d37\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f4296df1-e4d0-4720-9253-86eb030d2e3c\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:27:09.002804 3840 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-7wk8k ba3deabf586ef137b90a5512f3abe833987002fbef968c004e6b40639e320472 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-7wk8k ba3deabf586ef137b90a5512f3abe833987002fbef968c004e6b40639e320472 network default NAD default] I0326 23:27:09.012508 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-7wk8k, hostIfaceName: ba3deabf586ef13, network: default, NAD default, SandboxID: "ba3deabf586ef137b90a5512f3abe833987002fbef968c004e6b40639e320472", PCI device ID: , UID: "11c13115-9810-4554-a417-957f367ebb09", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0326 23:27:09.062657 3840 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-7wk8k ba3deabf586ef137b90a5512f3abe833987002fbef968c004e6b40639e320472 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-7wk8k ba3deabf586ef137b90a5512f3abe833987002fbef968c004e6b40639e320472 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:0e:07:9e:1b:33\",\"name\":\"ba3deabf586ef13\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bc640b2a-2f60-4a75-8735-672ae09696af\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:27:09.101896 3840 cni.go:403] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-rnshr 887fe815769a485b76b413f4c8154cac96ce029fef1144b566986dbae5fdadda network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-rnshr 887fe815769a485b76b413f4c8154cac96ce029fef1144b566986dbae5fdadda network default NAD default] I0326 23:27:09.111200 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-7988b548df-rnshr, hostIfaceName: 887fe815769a485, network: default, NAD default, SandboxID: "887fe815769a485b76b413f4c8154cac96ce029fef1144b566986dbae5fdadda", PCI device ID: , UID: "dd6f1830-ffae-457e-9179-a7788090f5f0", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0326 23:27:09.154833 3840 cni.go:424] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-rnshr 887fe815769a485b76b413f4c8154cac96ce029fef1144b566986dbae5fdadda network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-rnshr 887fe815769a485b76b413f4c8154cac96ce029fef1144b566986dbae5fdadda network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:41:ce:41:78:19\",\"name\":\"887fe815769a485\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ddf45811-0cfc-45c6-bfd7-b89d21fc82b9\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:27:12.622854 3840 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:27:12.622879 3840 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:27:12.622886 3840 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0326 23:27:13.158951 3840 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:13.158972 3840 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:13.158979 3840 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:13.195839 3840 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:13.195853 3840 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:13.195858 3840 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:13.195862 3840 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0326 23:27:14.830694 3840 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:14.831204 3840 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:14.852675 3840 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:14.852693 3840 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:14.861557 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0326 23:27:14.861598 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0326 23:27:20.548242 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=42 W0326 23:27:24.157572 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:24.157612 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:24.157618 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:24.157623 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0326 23:27:24.185049 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:24.185073 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:24.185082 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0326 23:27:29.794807 3840 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0326 23:27:29.817481 3840 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0326 23:27:31.289923 3840 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-45fkg] creating logical port openshift-pipelines_tekton-triggers-webhook-55d94ff6c6-45fkg for pod on switch ip-10-0-2-103.ec2.internal I0326 23:27:31.290025 3840 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-45fkg I0326 23:27:31.308932 3840 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-45fkg] pod update took 18.933115ms I0326 23:27:31.308951 3840 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-45fkg] addLogicalPort annotation time took 18.960426ms I0326 23:27:31.310124 3840 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-45fkg] addLogicalPort took 20.218504ms, libovsdb time 828.642µs I0326 23:27:31.648059 3840 cni.go:403] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-45fkg 2ca4e2f8065b43de60701b4b38db3e79b863ef5a6ad0a479ff5710caef118837 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-45fkg 2ca4e2f8065b43de60701b4b38db3e79b863ef5a6ad0a479ff5710caef118837 network default NAD default] I0326 23:27:31.656755 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-55d94ff6c6-45fkg, hostIfaceName: 2ca4e2f8065b43d, network: default, NAD default, SandboxID: "2ca4e2f8065b43de60701b4b38db3e79b863ef5a6ad0a479ff5710caef118837", PCI device ID: , UID: "820cea20-c8e2-42dc-a300-c3eb008f8bd7", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0326 23:27:31.703007 3840 cni.go:424] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-45fkg 2ca4e2f8065b43de60701b4b38db3e79b863ef5a6ad0a479ff5710caef118837 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-45fkg 2ca4e2f8065b43de60701b4b38db3e79b863ef5a6ad0a479ff5710caef118837 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:a9:53:2b:ed:af\",\"name\":\"2ca4e2f8065b43d\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1a61d59c-5680-47a1-9dbc-c76950a2175b\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:27:34.843386 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0326 23:27:43.643648 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0326 23:27:52.649376 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0326 23:27:52.751370 3840 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0326 23:27:53.054319 3840 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:53.054929 3840 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:27:53.073978 3840 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:53.073995 3840 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:27:53.081077 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0326 23:27:53.081099 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0326 23:27:55.830879 3840 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:27:55.830977 3840 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0326 23:27:55.843544 3840 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 12.592244ms I0326 23:27:55.843569 3840 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 12.623431ms I0326 23:27:55.845044 3840 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 14.18222ms, libovsdb time 1.018133ms I0326 23:28:02.204219 3840 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 8987c8a9557c3fe8f001e1f4c34837cf722a2f96bd209ccf1e823b4e1ce4abda network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 8987c8a9557c3fe8f001e1f4c34837cf722a2f96bd209ccf1e823b4e1ce4abda network default NAD default] I0326 23:28:02.212897 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 8987c8a9557c3fe, network: default, NAD default, SandboxID: "8987c8a9557c3fe8f001e1f4c34837cf722a2f96bd209ccf1e823b4e1ce4abda", PCI device ID: , UID: "12dd76c6-b1ec-4f3b-bd6a-e6cce320c0cf", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0326 23:28:02.259159 3840 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 8987c8a9557c3fe8f001e1f4c34837cf722a2f96bd209ccf1e823b4e1ce4abda network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 8987c8a9557c3fe8f001e1f4c34837cf722a2f96bd209ccf1e823b4e1ce4abda network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:3c:ed:dd:e8:43\",\"name\":\"8987c8a9557c3fe\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2db6fc35-1669-4cf8-8af6-fd355c15e45e\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:28:05.016067 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:05.016090 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:05.019208 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0326 23:28:07.453481 3840 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0326 23:28:13.970700 3840 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:28:13.970717 3840 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0326 23:28:13.970724 3840 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0326 23:28:24.368390 3840 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:24.368923 3840 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:25.131120 3840 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:25.131148 3840 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:25.139367 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:28:25.139389 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0326 23:28:25.168296 3840 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:28:25.168391 3840 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0326 23:28:25.191929 3840 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 23.570664ms I0326 23:28:25.191953 3840 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 23.600435ms I0326 23:28:25.193259 3840 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 24.979766ms, libovsdb time 892.173µs I0326 23:28:25.536378 3840 cni.go:403] [tsf-keycloak/keycloak-0 a1c90ef70336265eaa653ca1b412c4002c788f09fe5fb8b260a482afff196a6a network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 a1c90ef70336265eaa653ca1b412c4002c788f09fe5fb8b260a482afff196a6a network default NAD default] I0326 23:28:25.544784 3840 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: a1c90ef70336265, network: default, NAD default, SandboxID: "a1c90ef70336265eaa653ca1b412c4002c788f09fe5fb8b260a482afff196a6a", PCI device ID: , UID: "5faf2cc2-37eb-422d-aaec-b245c983aa24", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0326 23:28:25.589871 3840 cni.go:424] [tsf-keycloak/keycloak-0 a1c90ef70336265eaa653ca1b412c4002c788f09fe5fb8b260a482afff196a6a network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 a1c90ef70336265eaa653ca1b412c4002c788f09fe5fb8b260a482afff196a6a network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:f7:bf:3d:a1:38\",\"name\":\"a1c90ef70336265\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/75ce05a1-1411-4108-9a08-3861b53d8dbb\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:28:33.567146 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:28:33.567251 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:33.567266 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0326 23:28:39.336802 3840 namespace.go:142] [openshift-pipelines] updating namespace I0326 23:28:41.437630 3840 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-plnjj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:41.438412 3840 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-plnjj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:28:42.062872 3840 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0326 23:28:42.463697 3840 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-plnjj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:42.463725 3840 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-plnjj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:43.634340 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=5 W0326 23:28:46.836385 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:28:46.839204 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:46.839226 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak I0326 23:28:52.701168 3840 cni.go:403] [tsf-keycloak/keycloak-0 a1c90ef70336265eaa653ca1b412c4002c788f09fe5fb8b260a482afff196a6a network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 a1c90ef70336265eaa653ca1b412c4002c788f09fe5fb8b260a482afff196a6a network default NAD default] I0326 23:28:52.776261 3840 cni.go:424] [tsf-keycloak/keycloak-0 a1c90ef70336265eaa653ca1b412c4002c788f09fe5fb8b260a482afff196a6a network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 a1c90ef70336265eaa653ca1b412c4002c788f09fe5fb8b260a482afff196a6a network default NAD default], result "{}", err I0326 23:28:52.828729 3840 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:52.828758 3840 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0326 23:28:52.830109 3840 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.128.0.41 I0326 23:28:52.830136 3840 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:53.669400 3840 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:53.669424 3840 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:28:53.673972 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0326 23:28:53.674000 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0326 23:28:53.709442 3840 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:28:53.709550 3840 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0326 23:28:53.725602 3840 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 16.072383ms I0326 23:28:53.725626 3840 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 16.114749ms I0326 23:28:53.727116 3840 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 17.691253ms, libovsdb time 964.043µs I0326 23:28:54.071620 3840 cni.go:403] [tsf-keycloak/keycloak-0 2bc41de88dbc8d6cfb59948b65884eb4e7bb2590b7dc670873b8eb2dab71ed62 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 2bc41de88dbc8d6cfb59948b65884eb4e7bb2590b7dc670873b8eb2dab71ed62 network default NAD default] I0326 23:28:54.080462 3840 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 2bc41de88dbc8d6, network: default, NAD default, SandboxID: "2bc41de88dbc8d6cfb59948b65884eb4e7bb2590b7dc670873b8eb2dab71ed62", PCI device ID: , UID: "b7299f93-8f67-4708-9d55-64cb9a4a5d5f", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0326 23:28:54.127816 3840 cni.go:424] [tsf-keycloak/keycloak-0 2bc41de88dbc8d6cfb59948b65884eb4e7bb2590b7dc670873b8eb2dab71ed62 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 2bc41de88dbc8d6cfb59948b65884eb4e7bb2590b7dc670873b8eb2dab71ed62 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:80:76:0b:e0:d4\",\"name\":\"2bc41de88dbc8d6\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a72d4220-bbf3-4e67-9e1f-a8bebea583dd\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:28:55.494758 3840 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:28:55.495345 3840 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0326 23:28:55.700835 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:28:55.700900 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:28:55.700984 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:29:08.391604 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0326 23:29:08.391713 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0326 23:29:08.391725 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak I0326 23:29:09.548417 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=98 I0326 23:29:43.654024 3840 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0326 23:29:44.542702 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=50 I0326 23:29:45.659524 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0326 23:29:52.541300 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0326 23:29:55.598938 3840 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:55.598962 3840 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:29:55.604884 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:29:55.604909 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:29:59.623623 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0326 23:30:00.175468 3840 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29576130-nfdqk for pod on switch ip-10-0-2-103.ec2.internal I0326 23:30:00.175559 3840 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk I0326 23:30:00.189960 3840 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk] pod update took 14.423637ms I0326 23:30:00.189988 3840 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk] addLogicalPort annotation time took 14.456335ms I0326 23:30:00.191242 3840 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk] addLogicalPort took 15.789212ms, libovsdb time 885.094µs I0326 23:30:00.535001 3840 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk c2208b1591b82c6a45efc0994e3af306f9c2320d01c56c63e71d08d87013f7a4 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk c2208b1591b82c6a45efc0994e3af306f9c2320d01c56c63e71d08d87013f7a4 network default NAD default] I0326 23:30:00.543452 3840 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29576130-nfdqk, hostIfaceName: c2208b1591b82c6, network: default, NAD default, SandboxID: "c2208b1591b82c6a45efc0994e3af306f9c2320d01c56c63e71d08d87013f7a4", PCI device ID: , UID: "690b0424-02bf-4446-bcd0-feb2ad295d4c", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0326 23:30:00.588460 3840 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk c2208b1591b82c6a45efc0994e3af306f9c2320d01c56c63e71d08d87013f7a4 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk c2208b1591b82c6a45efc0994e3af306f9c2320d01c56c63e71d08d87013f7a4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:5b:9c:eb:97:04\",\"name\":\"c2208b1591b82c6\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a9d41cf8-b463-42c2-9790-ea11728fe697\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:30:04.054330 3840 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk c2208b1591b82c6a45efc0994e3af306f9c2320d01c56c63e71d08d87013f7a4 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk c2208b1591b82c6a45efc0994e3af306f9c2320d01c56c63e71d08d87013f7a4 network default NAD default] I0326 23:30:04.122838 3840 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk c2208b1591b82c6a45efc0994e3af306f9c2320d01c56c63e71d08d87013f7a4 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk c2208b1591b82c6a45efc0994e3af306f9c2320d01c56c63e71d08d87013f7a4 network default NAD default], result "{}", err I0326 23:30:04.171693 3840 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:04.171722 3840 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk I0326 23:30:04.173000 3840 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk, ips: 10.128.0.43 I0326 23:30:04.173024 3840 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:04.848862 3840 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:04.849464 3840 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:04.874610 3840 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:04.874632 3840 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:04.877521 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0326 23:30:04.877542 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0326 23:30:05.075092 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:05.075119 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576130-nfdqk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0326 23:30:14.509782 3840 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0326 23:30:19.538443 3840 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0326 23:30:20.920491 3840 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0326 23:30:23.980073 3840 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-jsgk9-ct7kc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:23.980667 3840 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-jsgk9-ct7kc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:30:25.000029 3840 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-jsgk9-ct7kc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:25.000054 3840 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-jsgk9-ct7kc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:30:33.659766 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0326 23:30:57.627665 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0326 23:31:00.192611 3840 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:00.193171 3840 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:00.235427 3840 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:00.235448 3840 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:00.240925 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0326 23:31:00.240947 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0326 23:31:09.259901 3840 obj_retry.go:464] Detected object tsf/patch-tekton-config-x9rfq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:09.260436 3840 obj_retry.go:464] Detected object tsf/patch-tekton-config-x9rfq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:10.282690 3840 obj_retry.go:458] Detected object tsf/patch-tekton-config-x9rfq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:10.282714 3840 obj_retry.go:458] Detected object tsf/patch-tekton-config-x9rfq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:10.548798 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=118 I0326 23:31:16.287699 3840 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-rxmtd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:16.288351 3840 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-rxmtd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:16.319314 3840 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-4ztw6-thqxc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:16.319822 3840 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-4ztw6-thqxc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:17.192555 3840 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-rxmtd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:17.192579 3840 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-rxmtd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:17.193911 3840 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-4ztw6-thqxc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:17.193932 3840 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-4ztw6-thqxc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:17.225536 3840 obj_retry.go:458] Detected object tsf/patch-tekton-config-x9rfq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:17.225552 3840 obj_retry.go:458] Detected object tsf/patch-tekton-config-x9rfq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:17.228862 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-x9rfq I0326 23:31:17.228885 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-x9rfq I0326 23:31:18.309166 3840 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-wgjsk-8brd5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:18.309753 3840 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-wgjsk-8brd5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:19.221165 3840 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-wgjsk-8brd5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:19.221187 3840 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-wgjsk-8brd5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:22.587770 3840 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-7df74f5f58-tklc5] creating logical port tsf-tas_ctlog-7df74f5f58-tklc5 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:31:22.587868 3840 kube.go:256] Updating pod tsf-tas/ctlog-7df74f5f58-tklc5 I0326 23:31:22.601657 3840 pod.go:62] [tsf-tas/ctlog-7df74f5f58-tklc5] pod update took 13.810384ms I0326 23:31:22.601679 3840 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-7df74f5f58-tklc5] addLogicalPort annotation time took 13.838146ms I0326 23:31:22.602970 3840 pods.go:271] [tsf-tas/ctlog-7df74f5f58-tklc5] addLogicalPort took 15.219024ms, libovsdb time 905.263µs I0326 23:31:22.944267 3840 cni.go:403] [tsf-tas/ctlog-7df74f5f58-tklc5 0a6e4247f0b0ecb0022cb35eb14874d5e82b8cf567a80a3dd2b7c3b5dbfa1332 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-7df74f5f58-tklc5 0a6e4247f0b0ecb0022cb35eb14874d5e82b8cf567a80a3dd2b7c3b5dbfa1332 network default NAD default] I0326 23:31:22.952983 3840 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-7df74f5f58-tklc5, hostIfaceName: 0a6e4247f0b0ecb, network: default, NAD default, SandboxID: "0a6e4247f0b0ecb0022cb35eb14874d5e82b8cf567a80a3dd2b7c3b5dbfa1332", PCI device ID: , UID: "add3cd0b-4765-4001-9fe9-8d1e08a680f6", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0326 23:31:22.998337 3840 cni.go:424] [tsf-tas/ctlog-7df74f5f58-tklc5 0a6e4247f0b0ecb0022cb35eb14874d5e82b8cf567a80a3dd2b7c3b5dbfa1332 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-7df74f5f58-tklc5 0a6e4247f0b0ecb0022cb35eb14874d5e82b8cf567a80a3dd2b7c3b5dbfa1332 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:36:47:36:18:d2\",\"name\":\"0a6e4247f0b0ecb\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/36c09e9b-fd34-4c0a-b4be-faf2b3c616c6\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:31:24.461394 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0326 23:31:27.253879 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0326 23:31:27.279832 3840 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0326 23:31:28.382408 3840 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:28.383052 3840 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:28.400609 3840 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:28.400629 3840 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:28.406455 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0326 23:31:28.406476 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines W0326 23:31:30.301693 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0326 23:31:32.427408 3840 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:32.427984 3840 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:32.453699 3840 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:32.453720 3840 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:32.461346 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0326 23:31:32.461367 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0326 23:31:33.713638 3840 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-bx6wq] creating logical port tsf-tpa_create-db-bx6wq for pod on switch ip-10-0-2-103.ec2.internal I0326 23:31:33.713724 3840 kube.go:256] Updating pod tsf-tpa/create-db-bx6wq I0326 23:31:33.730790 3840 pod.go:62] [tsf-tpa/create-db-bx6wq] pod update took 17.08887ms I0326 23:31:33.730813 3840 base_network_controller_pods.go:937] [default/tsf-tpa/create-db-bx6wq] addLogicalPort annotation time took 17.115065ms I0326 23:31:33.732230 3840 pods.go:271] [tsf-tpa/create-db-bx6wq] addLogicalPort took 18.606998ms, libovsdb time 972.311µs I0326 23:31:34.069922 3840 cni.go:403] [tsf-tpa/create-db-bx6wq d233f53c00a065be3f60d24b7e8f42139317198c1b7e4da0925de5b5e29c2ce1 network default NAD default] ADD starting CNI request [tsf-tpa/create-db-bx6wq d233f53c00a065be3f60d24b7e8f42139317198c1b7e4da0925de5b5e29c2ce1 network default NAD default] I0326 23:31:34.078162 3840 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-db-bx6wq, hostIfaceName: d233f53c00a065b, network: default, NAD default, SandboxID: "d233f53c00a065be3f60d24b7e8f42139317198c1b7e4da0925de5b5e29c2ce1", PCI device ID: , UID: "9983ca3c-8944-4b76-bd7b-8983ab588ae3", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0326 23:31:34.123764 3840 cni.go:424] [tsf-tpa/create-db-bx6wq d233f53c00a065be3f60d24b7e8f42139317198c1b7e4da0925de5b5e29c2ce1 network default NAD default] ADD finished CNI request [tsf-tpa/create-db-bx6wq d233f53c00a065be3f60d24b7e8f42139317198c1b7e4da0925de5b5e29c2ce1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:ef:d3:4d:15:9e\",\"name\":\"d233f53c00a065b\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/65135feb-9c96-4cc6-bc9b-5761031d7a08\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:31:35.474394 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0326 23:31:37.533303 3840 cni.go:403] [tsf-tpa/create-db-bx6wq d233f53c00a065be3f60d24b7e8f42139317198c1b7e4da0925de5b5e29c2ce1 network default NAD default] DEL starting CNI request [tsf-tpa/create-db-bx6wq d233f53c00a065be3f60d24b7e8f42139317198c1b7e4da0925de5b5e29c2ce1 network default NAD default] I0326 23:31:37.621908 3840 cni.go:424] [tsf-tpa/create-db-bx6wq d233f53c00a065be3f60d24b7e8f42139317198c1b7e4da0925de5b5e29c2ce1 network default NAD default] DEL finished CNI request [tsf-tpa/create-db-bx6wq d233f53c00a065be3f60d24b7e8f42139317198c1b7e4da0925de5b5e29c2ce1 network default NAD default], result "{}", err I0326 23:31:37.673181 3840 obj_retry.go:464] Detected object tsf-tpa/create-db-bx6wq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:37.673210 3840 pods.go:176] Deleting pod: tsf-tpa/create-db-bx6wq I0326 23:31:37.674610 3840 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-bx6wq, ips: 10.128.0.45 I0326 23:31:37.674638 3840 obj_retry.go:464] Detected object tsf-tpa/create-db-bx6wq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:38.696483 3840 obj_retry.go:458] Detected object tsf-tpa/create-db-bx6wq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:38.696516 3840 obj_retry.go:458] Detected object tsf-tpa/create-db-bx6wq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:38.797654 3840 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-bcw5h] creating logical port tsf-tpa_migrate-db-bcw5h for pod on switch ip-10-0-2-103.ec2.internal I0326 23:31:38.797786 3840 kube.go:256] Updating pod tsf-tpa/migrate-db-bcw5h I0326 23:31:38.810224 3840 pod.go:62] [tsf-tpa/migrate-db-bcw5h] pod update took 12.466621ms I0326 23:31:38.810248 3840 base_network_controller_pods.go:937] [default/tsf-tpa/migrate-db-bcw5h] addLogicalPort annotation time took 12.493347ms I0326 23:31:38.811603 3840 pods.go:271] [tsf-tpa/migrate-db-bcw5h] addLogicalPort took 13.964562ms, libovsdb time 846.927µs I0326 23:31:39.455996 3840 cni.go:403] [tsf-tpa/migrate-db-bcw5h dd5e6c56b47a45c87d93c47889ae516f470f328bb24f00522b02582b52096c1f network default NAD default] ADD starting CNI request [tsf-tpa/migrate-db-bcw5h dd5e6c56b47a45c87d93c47889ae516f470f328bb24f00522b02582b52096c1f network default NAD default] I0326 23:31:39.464223 3840 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-bcw5h, hostIfaceName: dd5e6c56b47a45c, network: default, NAD default, SandboxID: "dd5e6c56b47a45c87d93c47889ae516f470f328bb24f00522b02582b52096c1f", PCI device ID: , UID: "8ea3bb87-f2e5-4415-9f91-6015edf5e4c3", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0326 23:31:39.509041 3840 cni.go:424] [tsf-tpa/migrate-db-bcw5h dd5e6c56b47a45c87d93c47889ae516f470f328bb24f00522b02582b52096c1f network default NAD default] ADD finished CNI request [tsf-tpa/migrate-db-bcw5h dd5e6c56b47a45c87d93c47889ae516f470f328bb24f00522b02582b52096c1f network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:a3:47:3a:04:99\",\"name\":\"dd5e6c56b47a45c\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2f1de130-3aa9-4dd1-b94c-3662e261fb71\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:31:41.553806 3840 cni.go:403] [tsf-tpa/migrate-db-bcw5h dd5e6c56b47a45c87d93c47889ae516f470f328bb24f00522b02582b52096c1f network default NAD default] DEL starting CNI request [tsf-tpa/migrate-db-bcw5h dd5e6c56b47a45c87d93c47889ae516f470f328bb24f00522b02582b52096c1f network default NAD default] I0326 23:31:41.623937 3840 cni.go:424] [tsf-tpa/migrate-db-bcw5h dd5e6c56b47a45c87d93c47889ae516f470f328bb24f00522b02582b52096c1f network default NAD default] DEL finished CNI request [tsf-tpa/migrate-db-bcw5h dd5e6c56b47a45c87d93c47889ae516f470f328bb24f00522b02582b52096c1f network default NAD default], result "{}", err I0326 23:31:41.673674 3840 obj_retry.go:464] Detected object tsf-tpa/migrate-db-bcw5h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:41.673699 3840 pods.go:176] Deleting pod: tsf-tpa/migrate-db-bcw5h I0326 23:31:41.674996 3840 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-bcw5h, ips: 10.128.0.46 I0326 23:31:41.675025 3840 obj_retry.go:464] Detected object tsf-tpa/migrate-db-bcw5h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:42.696893 3840 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bcw5h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:42.696912 3840 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bcw5h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:45.812797 3840 base_network_controller_pods.go:487] [default/tsf-tpa/server-6d8d95b6cd-25c76] creating logical port tsf-tpa_server-6d8d95b6cd-25c76 for pod on switch ip-10-0-2-103.ec2.internal I0326 23:31:45.812944 3840 kube.go:256] Updating pod tsf-tpa/server-6d8d95b6cd-25c76 I0326 23:31:45.828902 3840 pod.go:62] [tsf-tpa/server-6d8d95b6cd-25c76] pod update took 15.987765ms I0326 23:31:45.828924 3840 base_network_controller_pods.go:937] [default/tsf-tpa/server-6d8d95b6cd-25c76] addLogicalPort annotation time took 16.017213ms I0326 23:31:45.830225 3840 pods.go:271] [tsf-tpa/server-6d8d95b6cd-25c76] addLogicalPort took 17.446045ms, libovsdb time 933.029µs I0326 23:31:45.851556 3840 base_network_controller_pods.go:487] [default/tsf-tpa/importer-74cd48f5c7-4q97h] creating logical port tsf-tpa_importer-74cd48f5c7-4q97h for pod on switch ip-10-0-2-103.ec2.internal I0326 23:31:45.851678 3840 kube.go:256] Updating pod tsf-tpa/importer-74cd48f5c7-4q97h I0326 23:31:45.873420 3840 pod.go:62] [tsf-tpa/importer-74cd48f5c7-4q97h] pod update took 21.7627ms I0326 23:31:45.873438 3840 base_network_controller_pods.go:937] [default/tsf-tpa/importer-74cd48f5c7-4q97h] addLogicalPort annotation time took 21.784805ms I0326 23:31:45.874778 3840 pods.go:271] [tsf-tpa/importer-74cd48f5c7-4q97h] addLogicalPort took 23.232895ms, libovsdb time 853.729µs I0326 23:31:47.115068 3840 obj_retry.go:464] Detected object tsf-tpa/create-importers-p4pv2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:47.115652 3840 obj_retry.go:464] Detected object tsf-tpa/create-importers-p4pv2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:47.553939 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=553 I0326 23:31:48.139467 3840 obj_retry.go:458] Detected object tsf-tpa/create-importers-p4pv2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:48.139490 3840 obj_retry.go:458] Detected object tsf-tpa/create-importers-p4pv2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:51.561855 3840 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-mf496-kzqmm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:51.562411 3840 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-mf496-kzqmm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:31:52.445122 3840 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-mf496-kzqmm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:52.445142 3840 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-mf496-kzqmm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:31:57.574266 3840 cni.go:403] [tsf-tpa/server-6d8d95b6cd-25c76 bc2c38988a9042d2260894486be342ad3eded05ce32fc0f40fdc245cc671d0da network default NAD default] ADD starting CNI request [tsf-tpa/server-6d8d95b6cd-25c76 bc2c38988a9042d2260894486be342ad3eded05ce32fc0f40fdc245cc671d0da network default NAD default] I0326 23:31:57.582863 3840 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: server-6d8d95b6cd-25c76, hostIfaceName: bc2c38988a9042d, network: default, NAD default, SandboxID: "bc2c38988a9042d2260894486be342ad3eded05ce32fc0f40fdc245cc671d0da", PCI device ID: , UID: "97e77102-6613-46d2-a6ba-1854f1283352", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0326 23:31:57.629923 3840 cni.go:424] [tsf-tpa/server-6d8d95b6cd-25c76 bc2c38988a9042d2260894486be342ad3eded05ce32fc0f40fdc245cc671d0da network default NAD default] ADD finished CNI request [tsf-tpa/server-6d8d95b6cd-25c76 bc2c38988a9042d2260894486be342ad3eded05ce32fc0f40fdc245cc671d0da network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:09:ad:03:59:fd\",\"name\":\"bc2c38988a9042d\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/467598e8-808e-44b9-b5c2-7e0666416a94\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:31:57.908854 3840 cni.go:403] [tsf-tpa/importer-74cd48f5c7-4q97h b2ff32125cef187cc6b6596ee5c1e1147d463f10a2e56a34b50f2ebb16aae395 network default NAD default] ADD starting CNI request [tsf-tpa/importer-74cd48f5c7-4q97h b2ff32125cef187cc6b6596ee5c1e1147d463f10a2e56a34b50f2ebb16aae395 network default NAD default] I0326 23:31:57.917461 3840 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: importer-74cd48f5c7-4q97h, hostIfaceName: b2ff32125cef187, network: default, NAD default, SandboxID: "b2ff32125cef187cc6b6596ee5c1e1147d463f10a2e56a34b50f2ebb16aae395", PCI device ID: , UID: "01a57185-e179-48dd-b5fe-3d5ea74cf6af", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0326 23:31:57.966429 3840 cni.go:424] [tsf-tpa/importer-74cd48f5c7-4q97h b2ff32125cef187cc6b6596ee5c1e1147d463f10a2e56a34b50f2ebb16aae395 network default NAD default] ADD finished CNI request [tsf-tpa/importer-74cd48f5c7-4q97h b2ff32125cef187cc6b6596ee5c1e1147d463f10a2e56a34b50f2ebb16aae395 network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:f6:1e:73:aa:69\",\"name\":\"b2ff32125cef187\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/155898e2-d61a-4b8c-a7be-b87da57f914f\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0326 23:31:58.652620 3840 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0326 23:31:59.659674 3840 util.go:959] Failed to build node endpoints for node ip-10-0-2-103.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server W0326 23:32:00.480558 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0326 23:32:16.707459 3840 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:16.708106 3840 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:16.728054 3840 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:16.728078 3840 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:16.734540 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0326 23:32:16.734568 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0326 23:32:17.465397 3840 namespace.go:98] [build-service] adding namespace I0326 23:32:17.466343 3840 namespace.go:102] [build-service] adding namespace took 921.204µs I0326 23:32:17.467381 3840 namespace.go:98] [image-controller] adding namespace I0326 23:32:17.468110 3840 namespace.go:102] [image-controller] adding namespace took 710.761µs I0326 23:32:17.468953 3840 namespace.go:98] [integration-service] adding namespace I0326 23:32:17.469657 3840 namespace.go:102] [integration-service] adding namespace took 687.183µs I0326 23:32:17.509801 3840 namespace.go:142] [build-service] updating namespace I0326 23:32:17.526937 3840 namespace.go:142] [build-service] updating namespace I0326 23:32:17.537549 3840 namespace.go:142] [image-controller] updating namespace I0326 23:32:17.552609 3840 namespace.go:142] [image-controller] updating namespace I0326 23:32:17.559184 3840 namespace.go:142] [integration-service] updating namespace I0326 23:32:17.574384 3840 namespace.go:142] [integration-service] updating namespace I0326 23:32:17.672914 3840 namespace.go:142] [build-service] updating namespace I0326 23:32:17.692825 3840 namespace.go:142] [integration-service] updating namespace I0326 23:32:17.717394 3840 namespace.go:98] [release-service] adding namespace I0326 23:32:17.718162 3840 namespace.go:102] [release-service] adding namespace took 729.276µs I0326 23:32:17.756842 3840 namespace.go:142] [konflux-ui] updating namespace I0326 23:32:17.764127 3840 namespace.go:142] [release-service] updating namespace I0326 23:32:17.775148 3840 namespace.go:142] [release-service] updating namespace I0326 23:32:17.812753 3840 namespace.go:98] [konflux-info] adding namespace I0326 23:32:17.813488 3840 namespace.go:102] [konflux-info] adding namespace took 719.421µs I0326 23:32:17.848816 3840 namespace.go:98] [namespace-lister] adding namespace I0326 23:32:17.849556 3840 namespace.go:102] [namespace-lister] adding namespace took 725.999µs I0326 23:32:17.866229 3840 namespace.go:142] [konflux-info] updating namespace I0326 23:32:17.915989 3840 namespace.go:142] [konflux-info] updating namespace I0326 23:32:17.923282 3840 namespace.go:142] [image-controller] updating namespace I0326 23:32:17.923316 3840 namespace.go:98] [enterprise-contract-service] adding namespace I0326 23:32:17.924188 3840 namespace.go:102] [enterprise-contract-service] adding namespace took 856.965µs I0326 23:32:17.946806 3840 namespace.go:142] [namespace-lister] updating namespace I0326 23:32:17.983378 3840 namespace.go:142] [namespace-lister] updating namespace I0326 23:32:18.014671 3840 namespace.go:98] [default-tenant] adding namespace I0326 23:32:18.014704 3840 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:32:18.015480 3840 namespace.go:102] [default-tenant] adding namespace took 791.223µs I0326 23:32:18.052820 3840 namespace.go:98] [segment-bridge] adding namespace I0326 23:32:18.053579 3840 namespace.go:102] [segment-bridge] adding namespace took 742.62µs I0326 23:32:18.065344 3840 namespace.go:142] [default-tenant] updating namespace I0326 23:32:18.065495 3840 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:32:18.177233 3840 namespace.go:142] [segment-bridge] updating namespace I0326 23:32:18.290959 3840 namespace.go:142] [default-tenant] updating namespace I0326 23:32:18.610548 3840 namespace.go:142] [segment-bridge] updating namespace I0326 23:32:18.884623 3840 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0326 23:32:18.885290 3840 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0326 23:32:18.885316 3840 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0326 23:32:18.888110 3840 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0326 23:32:18.934804 3840 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0326 23:32:18.934828 3840 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0326 23:32:18.935471 3840 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0326 23:32:19.723985 3840 namespace.go:142] [build-service] updating namespace I0326 23:32:20.047931 3840 namespace.go:142] [default-tenant] updating namespace I0326 23:32:20.364906 3840 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:32:20.790414 3840 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-c9nz8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:20.791053 3840 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-c9nz8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:20.896870 3840 namespace.go:142] [image-controller] updating namespace I0326 23:32:21.464627 3840 namespace.go:142] [integration-service] updating namespace I0326 23:32:21.821356 3840 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-c9nz8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:21.821378 3840 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-c9nz8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:21.851029 3840 namespace.go:142] [konflux-info] updating namespace I0326 23:32:22.112853 3840 namespace.go:142] [namespace-lister] updating namespace I0326 23:32:22.302737 3840 namespace.go:142] [release-service] updating namespace I0326 23:32:22.522070 3840 namespace.go:142] [segment-bridge] updating namespace I0326 23:32:22.611862 3840 namespace.go:142] [build-service] updating namespace I0326 23:32:22.616935 3840 namespace.go:142] [default-tenant] updating namespace I0326 23:32:22.638814 3840 namespace.go:142] [enterprise-contract-service] updating namespace I0326 23:32:22.670330 3840 namespace.go:142] [image-controller] updating namespace I0326 23:32:22.671928 3840 namespace.go:142] [integration-service] updating namespace I0326 23:32:22.691826 3840 namespace.go:142] [konflux-info] updating namespace I0326 23:32:22.725935 3840 namespace.go:142] [release-service] updating namespace I0326 23:32:22.725935 3840 namespace.go:142] [namespace-lister] updating namespace I0326 23:32:22.728860 3840 namespace.go:142] [segment-bridge] updating namespace I0326 23:32:22.778598 3840 namespace.go:142] [build-service] updating namespace I0326 23:32:22.855891 3840 namespace.go:142] [default-tenant] updating namespace W0326 23:32:22.885212 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0326 23:32:22.939843 3840 namespace.go:142] [enterprise-contract-service] updating namespace W0326 23:32:23.003300 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0326 23:32:23.018373 3840 namespace.go:142] [image-controller] updating namespace I0326 23:32:23.108894 3840 namespace.go:142] [integration-service] updating namespace I0326 23:32:23.229760 3840 namespace.go:142] [konflux-info] updating namespace I0326 23:32:23.327269 3840 namespace.go:142] [namespace-lister] updating namespace I0326 23:32:23.412258 3840 namespace.go:142] [release-service] updating namespace I0326 23:32:23.518678 3840 namespace.go:142] [segment-bridge] updating namespace W0326 23:32:24.656903 3840 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0326 23:32:33.719779 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0326 23:32:33.723226 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:32:33.750078 3840 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:32:33.750094 3840 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0326 23:32:37.261009 3840 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0326 23:32:37.287365 3840 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0326 23:32:37.287383 3840 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0326 23:32:37.306791 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0326 23:32:37.308397 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0326 23:32:37.329369 3840 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0326 23:32:37.329444 3840 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0326 23:32:45.058741 3840 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-bbf4j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:45.059300 3840 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-bbf4j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:32:45.319465 3840 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-bbf4j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:45.319486 3840 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-bbf4j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:32:45.325719 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-bbf4j I0326 23:32:45.325745 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-bbf4j I0326 23:32:58.646978 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0326 23:33:07.638685 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0326 23:33:19.626835 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0326 23:33:45.442198 3840 obj_retry.go:464] Detected object konflux-ui/proxy-7c4ffcc86b-bgxkr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:45.442785 3840 obj_retry.go:464] Detected object konflux-ui/proxy-7c4ffcc86b-bgxkr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:46.200537 3840 obj_retry.go:458] Detected object konflux-ui/proxy-7c4ffcc86b-bgxkr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:46.200561 3840 obj_retry.go:458] Detected object konflux-ui/proxy-7c4ffcc86b-bgxkr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:46.207269 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-7c4ffcc86b-bgxkr I0326 23:33:46.207292 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-7c4ffcc86b-bgxkr I0326 23:33:53.833167 3840 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:53.833797 3840 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:33:53.869191 3840 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:53.869212 3840 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:33:53.874111 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0326 23:33:53.874133 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0326 23:34:19.655621 3840 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf 5ce72f8230fb64d9dc2b274a04c09f6fa510f6eb8c84b0e38318268a4a74b130 network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf 5ce72f8230fb64d9dc2b274a04c09f6fa510f6eb8c84b0e38318268a4a74b130 network default NAD default] I0326 23:34:19.725802 3840 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf 5ce72f8230fb64d9dc2b274a04c09f6fa510f6eb8c84b0e38318268a4a74b130 network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf 5ce72f8230fb64d9dc2b274a04c09f6fa510f6eb8c84b0e38318268a4a74b130 network default NAD default], result "{}", err I0326 23:34:19.784869 3840 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:19.784905 3840 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf I0326 23:34:19.786275 3840 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf, ips: 10.128.0.30 I0326 23:34:19.786308 3840 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:20.414420 3840 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:20.414440 3840 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:20.418823 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf I0326 23:34:20.418846 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-wgpdf I0326 23:34:32.714219 3840 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-vdr4b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:32.714800 3840 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-vdr4b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:34:33.467105 3840 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-vdr4b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:33.467128 3840 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-vdr4b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:34:33.474820 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-vdr4b I0326 23:34:33.474843 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-vdr4b I0326 23:35:34.545506 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0326 23:35:41.663667 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0326 23:36:13.627201 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0326 23:36:57.638736 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0326 23:37:06.552296 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=68 I0326 23:37:06.631693 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0326 23:37:14.653893 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=11 I0326 23:37:17.657689 3840 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0326 23:38:20.546929 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=37 I0326 23:38:52.552408 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=69 I0326 23:39:25.557769 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=150 I0326 23:39:43.663699 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 I0326 23:39:54.630699 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0326 23:39:57.641975 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0326 23:40:03.809301 3840 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-8j8nb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:03.809910 3840 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-8j8nb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:40:04.690173 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-8j8nb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:04.690223 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576140-8j8nb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:40:39.552562 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=74 I0326 23:41:14.650515 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0326 23:41:38.049421 3840 obj_retry.go:458] Detected object tsf-tpa/create-db-bx6wq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:38.049447 3840 obj_retry.go:458] Detected object tsf-tpa/create-db-bx6wq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:38.055304 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-bx6wq I0326 23:41:38.055335 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-bx6wq I0326 23:41:42.030559 3840 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bcw5h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:42.030618 3840 obj_retry.go:458] Detected object tsf-tpa/migrate-db-bcw5h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:42.033428 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-bcw5h I0326 23:41:42.033459 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-bcw5h I0326 23:41:48.032088 3840 obj_retry.go:458] Detected object tsf-tpa/create-importers-p4pv2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:48.032139 3840 obj_retry.go:458] Detected object tsf-tpa/create-importers-p4pv2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:41:48.036395 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-p4pv2 I0326 23:41:48.036420 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-p4pv2 I0326 23:43:57.549864 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0326 23:44:23.667650 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0326 23:44:34.661796 3840 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0326 23:44:52.631712 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0326 23:45:28.658143 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0326 23:46:01.636165 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 W0326 23:46:14.941558 3840 duration.go:461] Config duration recorder: measurement expired for pod/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8bcmxc I0326 23:46:19.551195 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=9 I0326 23:46:28.667133 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0326 23:46:43.556038 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=44 I0326 23:46:48.642488 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:47:15.556099 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=6 I0326 23:47:21.635004 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0326 23:47:22.559681 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=9 I0326 23:48:27.770506 3840 namespace.go:98] [default-tenant-managed] adding namespace I0326 23:48:27.771513 3840 namespace.go:102] [default-tenant-managed] adding namespace took 983.382µs I0326 23:48:27.792020 3840 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:48:27.806555 3840 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:48:28.858186 3840 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:48:28.998547 3840 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:48:29.084240 3840 namespace.go:142] [default-tenant-managed] updating namespace I0326 23:49:17.882506 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:49:17.883078 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:49:18.561775 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=39 I0326 23:49:18.645581 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0326 23:49:31.671456 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0326 23:49:44.876210 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:49:44.876783 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:49:48.654763 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0326 23:50:05.070308 3840 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-ccctd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:05.070886 3840 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-ccctd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:05.922265 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-ccctd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:05.922292 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576150-ccctd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:05.960873 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:05.960890 3840 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:50:05.965539 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x I0326 23:50:05.965566 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29576120-x4f4x I0326 23:50:06.123777 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:50:06.124348 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:51:35.661802 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0326 23:51:59.666119 3840 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0326 23:52:14.977288 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:14.977875 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:30.553251 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0326 23:52:56.308404 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:56.308901 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:52:58.368516 3840 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod] creating logical port default-tenant_tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod for pod on switch ip-10-0-2-103.ec2.internal I0326 23:52:58.368672 3840 kube.go:256] Updating pod default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod I0326 23:52:58.395441 3840 pod.go:62] [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod] pod update took 26.811281ms I0326 23:52:58.395465 3840 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod] addLogicalPort annotation time took 26.841938ms I0326 23:52:58.396944 3840 pods.go:271] [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod] addLogicalPort took 28.420117ms, libovsdb time 1.043049ms I0326 23:52:58.730930 3840 cni.go:403] [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod 633e01abb32eb06b4f7ac0a9e82219876405e5e93baac726acb8564fbefe3c19 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod 633e01abb32eb06b4f7ac0a9e82219876405e5e93baac726acb8564fbefe3c19 network default NAD default] I0326 23:52:58.740138 3840 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod, hostIfaceName: 633e01abb32eb06, network: default, NAD default, SandboxID: "633e01abb32eb06b4f7ac0a9e82219876405e5e93baac726acb8564fbefe3c19", PCI device ID: , UID: "cdd4b413-6bca-466e-ae35-3c30ba214676", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0326 23:52:58.786625 3840 cni.go:424] [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod 633e01abb32eb06b4f7ac0a9e82219876405e5e93baac726acb8564fbefe3c19 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod 633e01abb32eb06b4f7ac0a9e82219876405e5e93baac726acb8564fbefe3c19 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:23:a2:f2:90:96\",\"name\":\"633e01abb32eb06\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/482902de-8b8f-47c0-91ce-0dcc3a756a96\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0326 23:53:19.636185 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0326 23:53:49.640402 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0326 23:53:51.566734 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=12 I0326 23:53:53.554367 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8 I0326 23:53:55.701497 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:55.702086 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:56.779785 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:56.780332 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:56.811948 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:56.812439 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:58.867166 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:53:58.867708 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:14.523896 3840 cni.go:403] [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod 633e01abb32eb06b4f7ac0a9e82219876405e5e93baac726acb8564fbefe3c19 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod 633e01abb32eb06b4f7ac0a9e82219876405e5e93baac726acb8564fbefe3c19 network default NAD default] I0326 23:54:14.602145 3840 cni.go:424] [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod 633e01abb32eb06b4f7ac0a9e82219876405e5e93baac726acb8564fbefe3c19 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod 633e01abb32eb06b4f7ac0a9e82219876405e5e93baac726acb8564fbefe3c19 network default NAD default], result "{}", err I0326 23:54:14.681122 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:14.681216 3840 pods.go:176] Deleting pod: default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod I0326 23:54:14.682668 3840 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod, ips: 10.128.0.49 I0326 23:54:14.682700 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:14.844938 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:14.845475 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:20.565378 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=182 I0326 23:54:24.012278 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:24.012862 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:24.055288 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:24.055821 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:30.646495 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0326 23:54:43.204897 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:43.205463 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:44.579957 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=11 I0326 23:54:46.670729 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0326 23:54:50.281679 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:54:50.282288 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:55:31.639299 3840 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0326 23:56:17.643353 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:17.643918 3840 obj_retry.go:464] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:17.688702 3840 obj_retry.go:458] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:17.688762 3840 obj_retry.go:458] Detected object default-tenant/tsf-comp-azfv-on-push-xxcfp-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:56:28.560290 3840 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=67 I0326 23:56:43.103863 3840 obj_retry.go:464] Detected object default-tenant/my-integration-test-pyer-2nlzs-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:56:43.104423 3840 obj_retry.go:464] Detected object default-tenant/my-integration-test-pyer-2nlzs-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:09.648958 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0326 23:57:13.308783 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-verify-access-to-resources-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:13.309426 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-verify-access-to-resources-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:34.536309 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-collect-data-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:34.537012 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-collect-data-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:41.658184 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0326 23:57:44.688772 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-collect-task-params-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:44.689312 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-collect-task-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:44.745924 3840 obj_retry.go:458] Detected object default-tenant-managed/managed-5tk5z-collect-task-params-pod of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:57:44.745947 3840 obj_retry.go:458] Detected object default-tenant-managed/managed-5tk5z-collect-task-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:57:45.693007 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-collect-registry-token-secret-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:45.693528 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-collect-registry-token-secret-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:58.814207 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-check-data-keys-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:57:58.814765 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-check-data-keys-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:05.885251 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-reduce-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:05.885829 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-reduce-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:29.099203 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-apply-mapping-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:29.099779 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-apply-mapping-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:37.615029 3840 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-91bd6489b9-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:37.615625 3840 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-91bd6489b9-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:38.191274 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-filter-already-released-images-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:38.191827 3840 obj_retry.go:464] Detected object default-tenant-managed/managed-5tk5z-filter-already-released-images-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0326 23:58:38.352282 3840 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-91bd6489b9-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:58:38.352307 3840 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-91bd6489b9-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0326 23:58:38.355089 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant-managed/affinity-assistant-91bd6489b9-0 I0326 23:58:38.355114 3840 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod default-tenant-managed/affinity-assistant-91bd6489b9-0 I0326 23:58:46.675068 3840 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11