+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-99.ec2.internal ++ [[ -n ip-10-0-0-99.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-99.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' I0428 15:35:36.755304274 - disable conntrack on geneve port + echo 'I0428 15:35:36.755304274 - disable conntrack on geneve port' + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0428 15:35:36.785515595 - starting ovnkube-node' I0428 15:35:36.785515595 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-99.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-99.ec2.internal --init-node ip-10-0-0-99.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-99.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0428 15:35:36.834177 3815 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0428 15:35:36.834270 3815 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-c7f8c090a1.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0428 15:35:36.836732 3815 kube.go:431] Waiting for certificate I0428 15:35:36.836760 3815 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0428 15:35:36.836798 3815 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0428 15:35:36.837631 3815 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 15:35:36.875899 3815 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:35:36.875921 3815 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:35:36.878803 3815 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:35:36.878891 3815 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-77d6d" I0428 15:35:36.882381 3815 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-77d6d" I0428 15:35:36.882447 3815 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:35:37.836896 3815 kube.go:438] Certificate found I0428 15:35:37.837552 3815 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 15:35:37.838542 3815 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0428 15:35:37.890884 3815 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-29 15:30:36 +0000 UTC" deadline="2026-04-29 09:15:35.106624012 +0000 UTC" I0428 15:35:37.890917 3815 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="17h39m57.215716418s" I0428 15:35:37.908504 3815 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 15:35:37.925033 3815 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0428 15:35:37.925192 3815 factory.go:561] Starting watch factory I0428 15:35:37.925350 3815 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.925400 3815 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.925640 3815 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.925666 3815 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.925923 3815 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.925944 3815 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.926340 3815 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.926389 3815 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.926728 3815 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.926792 3815 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.927022 3815 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.927066 3815 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.929059 3815 controller_manager.go:370] Starting the ovnkube controller I0428 15:35:37.929128 3815 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-99.ec2.internal I0428 15:35:37.929220 3815 controller_manager.go:395] NBDB zone sync took: 72.871µs I0428 15:35:37.929233 3815 factory.go:561] Starting watch factory I0428 15:35:37.931746 3815 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0428 15:35:37.931804 3815 server.go:101] MetricServer registers OVS metrics I0428 15:35:37.932060 3815 server.go:105] MetricServer registers OVN DB metrics I0428 15:35:37.936677 3815 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0428 15:35:37.936704 3815 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0428 15:35:37.945660 3815 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.945877 3815 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.950187 3815 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.950799 3815 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.951230 3815 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.951932 3815 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0428 15:35:37.951992 3815 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0428 15:35:37.956178 3815 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:35:37.957009 3815 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0428 15:35:37.957050 3815 server.go:109] MetricServer registers OVN Controller metrics I0428 15:35:37.961950 3815 server.go:113] MetricServer registers OVN Northd metrics I0428 15:35:37.965865 3815 metrics.go:519] OVN Metrics Server starts to run ... I0428 15:35:38.030769 3815 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 15:35:38.030798 3815 factory.go:1977] *v1.Node informer cache synced successfully I0428 15:35:38.030804 3815 factory.go:1977] *v1.Service informer cache synced successfully I0428 15:35:38.030809 3815 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 15:35:38.030814 3815 factory.go:1977] *v1.Pod informer cache synced successfully I0428 15:35:38.030819 3815 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 15:35:38.030925 3815 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:35:38.030950 3815 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:35:38.030926 3815 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:35:38.030980 3815 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:35:38.034937 3815 factory.go:1977] *v1.Service informer cache synced successfully I0428 15:35:38.034947 3815 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 15:35:38.034951 3815 factory.go:1977] *v1.Pod informer cache synced successfully I0428 15:35:38.034954 3815 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 15:35:38.034957 3815 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 15:35:38.034960 3815 factory.go:1977] *v1.Node informer cache synced successfully I0428 15:35:38.038304 3815 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:35:38.038304 3815 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:35:38.039987 3815 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 15:35:38.039999 3815 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 15:35:38.040055 3815 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.040068 3815 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.046964 3815 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.136299 3815 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 15:35:38.136335 3815 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 15:35:38.141498 3815 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 15:35:38.141593 3815 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.141612 3815 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.144917 3815 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.146825 3815 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 15:35:38.146834 3815 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 15:35:38.146896 3815 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.146908 3815 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.149624 3815 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.152170 3815 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 15:35:38.152195 3815 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 15:35:38.152282 3815 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.152296 3815 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.154974 3815 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.157958 3815 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 15:35:38.158016 3815 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.158028 3815 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.158050 3815 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 15:35:38.161001 3815 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.163026 3815 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 15:35:38.163190 3815 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:35:38.163204 3815 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:35:38.163192 3815 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 15:35:38.165633 3815 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:35:38.168564 3815 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 15:35:38.168762 3815 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 15:35:38.168794 3815 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.168811 3815 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.168917 3815 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.168931 3815 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.173182 3815 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.173222 3815 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:35:38.174068 3815 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 15:35:38.174077 3815 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 15:35:38.174084 3815 factory.go:680] Watch Factory start up complete, took: 248.897282ms I0428 15:35:38.174085 3815 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 15:35:38.174171 3815 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0428 15:35:38.179767 3815 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 15:35:38.179777 3815 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 15:35:38.179784 3815 factory.go:680] Watch Factory start up complete, took: 250.551092ms I0428 15:35:38.179791 3815 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-0-99.ec2.internal" zone I0428 15:35:38.222261 3815 default_node_network_controller.go:710] Initializing the default node network controller I0428 15:35:38.294706 3815 udn_isolation.go:101] Starting UDN host isolation manager I0428 15:35:38.302433 3815 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0428 15:35:38.362386 3815 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0428 15:35:38.362456 3815 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0428 15:35:38.362471 3815 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0428 15:35:38.396587 3815 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0428 15:35:38.396679 3815 default_node_network_controller.go:809] Node ip-10-0-0-99.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0428 15:35:38.402002 3815 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.99"] k8s.ovn.org/zone-name:ip-10-0-0-99.ec2.internal] on node ip-10-0-0-99.ec2.internal I0428 15:35:38.425647 3815 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0428 15:35:38.429931 3815 controller_manager.go:429] Waiting for node in zone sync took: 250.128515ms I0428 15:35:38.432158 3815 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0428 15:35:38.432341 3815 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0428 15:35:38.432475 3815 gateway_init.go:263] Preparing Gateway I0428 15:35:38.432487 3815 gateway_shared_intf.go:1687] Creating new gateway I0428 15:35:38.435639 3815 controller_manager.go:611] Switching to transit router for layer2 networks I0428 15:35:38.435655 3815 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-99.ec2.internal I0428 15:35:38.449836 3815 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 15:35:38.456467 3815 controller_manager.go:290] SCTP support detected in OVN I0428 15:35:38.457080 3815 services_controller.go:69] Creating services controller for network=default I0428 15:35:38.457178 3815 udn_enabled_svc.go:80] Waiting for service informer to sync I0428 15:35:38.457217 3815 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0428 15:35:38.457273 3815 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0428 15:35:38.457291 3815 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0428 15:35:38.457303 3815 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 15:35:38.457474 3815 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0428 15:35:38.457518 3815 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0428 15:35:38.457555 3815 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 43.094µs I0428 15:35:38.457563 3815 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 35.503µs I0428 15:35:38.457572 3815 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 52.518µs I0428 15:35:38.457579 3815 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0428 15:35:38.457592 3815 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0428 15:35:38.457604 3815 default_network_controller.go:325] Starting the default network controller I0428 15:35:38.457637 3815 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 14.229µs I0428 15:35:38.457641 3815 udn_enabled_svc.go:95] Performing full resync I0428 15:35:38.457898 3815 udn_enabled_svc.go:99] Waiting for handler to sync I0428 15:35:38.457910 3815 udn_enabled_svc.go:104] Starting worker I0428 15:35:38.466958 3815 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0428 15:35:38.467023 3815 repair.go:33] Syncing exgw routes took 52.958µs I0428 15:35:38.467050 3815 default_network_controller.go:414] Starting all the Watchers... I0428 15:35:38.467198 3815 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0428 15:35:38.467211 3815 namespace.go:98] [openshift-insights] adding namespace I0428 15:35:38.467218 3815 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0428 15:35:38.467216 3815 namespace.go:98] [openshift-ingress] adding namespace I0428 15:35:38.467224 3815 namespace.go:98] [open-cluster-management-2pv84gmg8k1purftr303td5tpat9d92k] adding namespace I0428 15:35:38.467252 3815 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0428 15:35:38.467254 3815 namespace.go:98] [openshift-authentication-operator] adding namespace I0428 15:35:38.467264 3815 namespace.go:98] [openshift] adding namespace I0428 15:35:38.467265 3815 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0428 15:35:38.467273 3815 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0428 15:35:38.467274 3815 namespace.go:98] [openshift-controller-manager] adding namespace I0428 15:35:38.467277 3815 namespace.go:98] [openshift-customer-monitoring] adding namespace I0428 15:35:38.467277 3815 namespace.go:98] [openshift-backplane-csm] adding namespace I0428 15:35:38.467296 3815 namespace.go:98] [kube-public] adding namespace I0428 15:35:38.467197 3815 namespace.go:98] [openshift-package-operator] adding namespace I0428 15:35:38.467759 3815 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 547.73µs I0428 15:35:38.467774 3815 namespace.go:98] [openshift-backplane-srep] adding namespace I0428 15:35:38.468190 3815 namespace.go:102] [openshift-backplane-srep] adding namespace took 408.49µs I0428 15:35:38.468208 3815 namespace.go:98] [openshift-config-operator] adding namespace I0428 15:35:38.468725 3815 namespace.go:102] [openshift-insights] adding namespace took 1.502202ms I0428 15:35:38.468740 3815 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0428 15:35:38.469211 3815 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 464.093µs I0428 15:35:38.469228 3815 namespace.go:98] [openshift-backplane-cee] adding namespace I0428 15:35:38.469715 3815 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 2.489414ms I0428 15:35:38.469726 3815 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0428 15:35:38.470154 3815 namespace.go:102] [openshift-ingress] adding namespace took 2.923422ms I0428 15:35:38.470162 3815 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0428 15:35:38.470652 3815 namespace.go:102] [open-cluster-management-2pv84gmg8k1purftr303td5tpat9d92k] adding namespace took 3.417343ms I0428 15:35:38.470661 3815 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0428 15:35:38.471109 3815 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 3.852216ms I0428 15:35:38.471118 3815 namespace.go:98] [openshift-kube-scheduler] adding namespace I0428 15:35:38.471564 3815 namespace.go:102] [openshift-authentication-operator] adding namespace took 4.298789ms I0428 15:35:38.471584 3815 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0428 15:35:38.472032 3815 namespace.go:102] [openshift] adding namespace took 4.760085ms I0428 15:35:38.472043 3815 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0428 15:35:38.472538 3815 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 5.26244ms I0428 15:35:38.472553 3815 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0428 15:35:38.473012 3815 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 5.73299ms I0428 15:35:38.473021 3815 namespace.go:98] [openshift-infra] adding namespace I0428 15:35:38.473432 3815 namespace.go:102] [openshift-controller-manager] adding namespace took 6.151517ms I0428 15:35:38.473441 3815 namespace.go:98] [openshift-ingress-canary] adding namespace I0428 15:35:38.473964 3815 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.678185ms I0428 15:35:38.473980 3815 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0428 15:35:38.474424 3815 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.135691ms I0428 15:35:38.474434 3815 namespace.go:98] [openshift-image-registry] adding namespace I0428 15:35:38.474951 3815 namespace.go:102] [kube-public] adding namespace took 7.642347ms I0428 15:35:38.474966 3815 namespace.go:98] [openshift-must-gather-operator] adding namespace I0428 15:35:38.475395 3815 namespace.go:102] [openshift-package-operator] adding namespace took 8.049618ms I0428 15:35:38.475409 3815 namespace.go:98] [openshift-network-console] adding namespace I0428 15:35:38.475794 3815 namespace.go:102] [openshift-config-operator] adding namespace took 7.581212ms I0428 15:35:38.475803 3815 namespace.go:98] [openshift-machine-api] adding namespace I0428 15:35:38.476154 3815 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.920012ms I0428 15:35:38.476174 3815 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0428 15:35:38.476578 3815 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.847614ms I0428 15:35:38.476591 3815 namespace.go:98] [openshift-backplane-tam] adding namespace I0428 15:35:38.476949 3815 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.783026ms I0428 15:35:38.476958 3815 namespace.go:98] [openshift-network-diagnostics] adding namespace I0428 15:35:38.477326 3815 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.660464ms I0428 15:35:38.477339 3815 namespace.go:98] [openshift-etcd] adding namespace I0428 15:35:38.477689 3815 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.566741ms I0428 15:35:38.477700 3815 namespace.go:98] [openshift-host-network] adding namespace I0428 15:35:38.478077 3815 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.487402ms I0428 15:35:38.478089 3815 namespace.go:98] [openshift-node] adding namespace I0428 15:35:38.478489 3815 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.44221ms I0428 15:35:38.478502 3815 namespace.go:98] [openshift-monitoring] adding namespace I0428 15:35:38.478822 3815 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.264718ms I0428 15:35:38.478833 3815 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0428 15:35:38.479262 3815 namespace.go:102] [openshift-infra] adding namespace took 6.234846ms I0428 15:35:38.479280 3815 namespace.go:98] [default] adding namespace I0428 15:35:38.479731 3815 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.285758ms I0428 15:35:38.479743 3815 namespace.go:98] [kube-system] adding namespace I0428 15:35:38.480138 3815 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.150079ms I0428 15:35:38.480158 3815 namespace.go:98] [openshift-console-operator] adding namespace I0428 15:35:38.480594 3815 namespace.go:102] [openshift-image-registry] adding namespace took 6.154148ms I0428 15:35:38.480610 3815 namespace.go:98] [openshift-multus] adding namespace I0428 15:35:38.481002 3815 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.03052ms I0428 15:35:38.481020 3815 namespace.go:98] [openshift-config] adding namespace I0428 15:35:38.481409 3815 namespace.go:102] [openshift-network-console] adding namespace took 5.993248ms I0428 15:35:38.481425 3815 namespace.go:98] [openshift-console-user-settings] adding namespace I0428 15:35:38.481797 3815 namespace.go:102] [openshift-machine-api] adding namespace took 5.987902ms I0428 15:35:38.481812 3815 namespace.go:98] [openshift-dns-operator] adding namespace I0428 15:35:38.482183 3815 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.002626ms I0428 15:35:38.482199 3815 namespace.go:98] [openshift-cloud-credential-operator] adding namespace W0428 15:35:38.482556 3815 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-mt5ft: pod openshift-network-diagnostics/network-check-target-mt5ft: no pod IPs found I0428 15:35:38.482600 3815 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.004168ms I0428 15:35:38.482617 3815 namespace.go:98] [openshift-backplane-cse] adding namespace I0428 15:35:38.483046 3815 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.082834ms I0428 15:35:38.483059 3815 namespace.go:98] [openshift-logging] adding namespace I0428 15:35:38.483450 3815 namespace.go:102] [openshift-etcd] adding namespace took 6.104369ms I0428 15:35:38.483463 3815 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0428 15:35:38.483919 3815 namespace.go:102] [openshift-host-network] adding namespace took 6.212527ms I0428 15:35:38.483937 3815 namespace.go:98] [dedicated-admin] adding namespace I0428 15:35:38.484270 3815 namespace.go:102] [openshift-node] adding namespace took 6.175883ms I0428 15:35:38.484287 3815 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0428 15:35:38.484712 3815 namespace.go:102] [openshift-monitoring] adding namespace took 6.204566ms I0428 15:35:38.484726 3815 namespace.go:98] [openshift-operators] adding namespace I0428 15:35:38.485103 3815 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.265104ms I0428 15:35:38.485116 3815 namespace.go:98] [openshift-operators-redhat] adding namespace I0428 15:35:38.485494 3815 namespace.go:102] [default] adding namespace took 6.208432ms I0428 15:35:38.485505 3815 namespace.go:98] [kube-node-lease] adding namespace I0428 15:35:38.486040 3815 namespace.go:102] [kube-system] adding namespace took 6.290199ms I0428 15:35:38.486056 3815 namespace.go:98] [openshift-cluster-samples-operator] adding namespace W0428 15:35:38.486586 3815 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-gml9f: pod openshift-multus/network-metrics-daemon-gml9f: no pod IPs found I0428 15:35:38.486616 3815 namespace.go:102] [openshift-console-operator] adding namespace took 6.451268ms I0428 15:35:38.486629 3815 namespace.go:98] [openshift-route-controller-manager] adding namespace I0428 15:35:38.487094 3815 namespace.go:102] [openshift-multus] adding namespace took 6.47712ms I0428 15:35:38.487108 3815 namespace.go:98] [openshift-config-managed] adding namespace I0428 15:35:38.487470 3815 namespace.go:102] [openshift-config] adding namespace took 6.444018ms I0428 15:35:38.487481 3815 namespace.go:98] [openshift-ingress-operator] adding namespace I0428 15:35:38.487878 3815 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.447049ms I0428 15:35:38.487891 3815 namespace.go:98] [openshift-marketplace] adding namespace I0428 15:35:38.488313 3815 namespace.go:102] [openshift-dns-operator] adding namespace took 6.495134ms I0428 15:35:38.488700 3815 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.495452ms I0428 15:35:38.488715 3815 namespace.go:98] [openshift-service-ca-operator] adding namespace I0428 15:35:38.489081 3815 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.458778ms I0428 15:35:38.489093 3815 namespace.go:98] [openshift-apiserver-operator] adding namespace I0428 15:35:38.489446 3815 namespace.go:102] [openshift-logging] adding namespace took 6.381778ms I0428 15:35:38.489457 3815 namespace.go:98] [openshift-apiserver] adding namespace I0428 15:35:38.489886 3815 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.418216ms I0428 15:35:38.489898 3815 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0428 15:35:38.490215 3815 namespace.go:102] [dedicated-admin] adding namespace took 6.272262ms I0428 15:35:38.490227 3815 namespace.go:98] [openshift-dns] adding namespace I0428 15:35:38.490691 3815 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.398253ms I0428 15:35:38.490702 3815 namespace.go:98] [openshift-cluster-version] adding namespace I0428 15:35:38.491049 3815 namespace.go:102] [openshift-operators] adding namespace took 6.316743ms I0428 15:35:38.491063 3815 namespace.go:98] [openshift-authentication] adding namespace I0428 15:35:38.491536 3815 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.415817ms I0428 15:35:38.491549 3815 namespace.go:98] [openshift-kube-apiserver] adding namespace I0428 15:35:38.491871 3815 namespace.go:102] [kube-node-lease] adding namespace took 6.360982ms I0428 15:35:38.491882 3815 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0428 15:35:38.492330 3815 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.268811ms I0428 15:35:38.492347 3815 namespace.go:98] [openshift-backplane] adding namespace I0428 15:35:38.492739 3815 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.104516ms I0428 15:35:38.492753 3815 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0428 15:35:38.493126 3815 namespace.go:102] [openshift-config-managed] adding namespace took 6.011987ms I0428 15:35:38.493547 3815 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.059994ms I0428 15:35:38.493565 3815 namespace.go:98] [openshift-machine-config-operator] adding namespace I0428 15:35:38.493940 3815 namespace.go:102] [openshift-marketplace] adding namespace took 6.04364ms I0428 15:35:38.493954 3815 namespace.go:98] [openshift-console] adding namespace I0428 15:35:38.494310 3815 namespace.go:102] [openshift-service-ca-operator] adding namespace took 5.589363ms I0428 15:35:38.494756 3815 namespace.go:102] [openshift-apiserver-operator] adding namespace took 5.657429ms I0428 15:35:38.495149 3815 namespace.go:102] [openshift-apiserver] adding namespace took 5.686033ms I0428 15:35:38.495643 3815 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.738856ms I0428 15:35:38.496161 3815 namespace.go:102] [openshift-dns] adding namespace took 5.929017ms I0428 15:35:38.496704 3815 namespace.go:102] [openshift-cluster-version] adding namespace took 5.995481ms I0428 15:35:38.496718 3815 namespace.go:98] [openshift-backplane-mobb] adding namespace I0428 15:35:38.497265 3815 namespace.go:102] [openshift-authentication] adding namespace took 6.196756ms I0428 15:35:38.497280 3815 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0428 15:35:38.497662 3815 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.107802ms I0428 15:35:38.498073 3815 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.186628ms I0428 15:35:38.498085 3815 namespace.go:98] [openshift-network-node-identity] adding namespace I0428 15:35:38.498494 3815 namespace.go:102] [openshift-backplane] adding namespace took 6.140144ms I0428 15:35:38.498870 3815 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.111751ms I0428 15:35:38.499270 3815 namespace.go:102] [openshift-machine-config-operator] adding namespace took 5.698179ms I0428 15:35:38.499284 3815 namespace.go:98] [openshift-network-operator] adding namespace I0428 15:35:38.499714 3815 namespace.go:102] [openshift-console] adding namespace took 5.756561ms I0428 15:35:38.500081 3815 namespace.go:102] [openshift-backplane-mobb] adding namespace took 3.358731ms I0428 15:35:38.500496 3815 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 3.20878ms I0428 15:35:38.500866 3815 namespace.go:102] [openshift-network-node-identity] adding namespace took 2.775233ms I0428 15:35:38.501197 3815 namespace.go:102] [openshift-network-operator] adding namespace took 1.908299ms W0428 15:35:38.501308 3815 master.go:458] Unable to parse local node ip-10-0-0-99.ec2.internal chassis-id annotation. Chassis may be removed during sync I0428 15:35:38.501320 3815 master.go:481] Removing stale chassis with ID/Name: 5ccccf10-2a2a-4556-8e67-bbda603851ad, hostname: ip-10-0-0-99.ec2.internal I0428 15:35:38.501755 3815 master.go:548] Adding or Updating local node "ip-10-0-0-99.ec2.internal" for network "default" I0428 15:35:38.503295 3815 zone_ic_handler.go:195] Time taken to create transit switch: 836.353µs I0428 15:35:38.503312 3815 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-15.ec2.internal for the network default I0428 15:35:38.504558 3815 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-15.ec2.internal" on network "default" took: 2.111121ms I0428 15:35:38.505669 3815 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-99.ec2.internal for the network default I0428 15:35:38.505835 3815 zone_ic_handler.go:195] Time taken to create transit switch: 153.678µs I0428 15:35:38.506495 3815 default_network_controller.go:823] Node add failed for ip-10-0-0-99.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-99.ec2.internal, error getting gateway config for node ip-10-0-0-99.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-99.ec2.internal", failed to update chassis to local for local node ip-10-0-0-99.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-99.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-99.ec2.internal] E0428 15:35:38.506553 3815 obj_retry.go:575] Failed to create *v1.Node ip-10-0-0-99.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-99.ec2.internal, error getting gateway config for node ip-10-0-0-99.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-99.ec2.internal", failed to update chassis to local for local node ip-10-0-0-99.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-99.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-99.ec2.internal] I0428 15:35:38.506610 3815 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0428 15:35:38.506616 3815 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-99.ec2.internal", UID:"68525b23-a40f-4352-8e68-63f1b116acdf", APIVersion:"v1", ResourceVersion:"8528", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-99.ec2.internal, error getting gateway config for node ip-10-0-0-99.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-99.ec2.internal", failed to update chassis to local for local node ip-10-0-0-99.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-99.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-99.ec2.internal] I0428 15:35:38.506633 3815 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0428 15:35:38.506650 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-99.ec2.internal in network "default" I0428 15:35:38.506656 3815 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0428 15:35:38.506674 3815 node_tracker.go:235] Node ip-10-0-0-99.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-99.ec2.internal" I0428 15:35:38.506687 3815 node_tracker.go:169] Node ip-10-0-0-99.ec2.internal switch + router changed, syncing services in network "default" I0428 15:35:38.506702 3815 services_controller.go:574] Full service sync requested for network=default I0428 15:35:38.506712 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-15.ec2.internal in network "default" I0428 15:35:38.506799 3815 node_tracker.go:169] Node ip-10-0-2-15.ec2.internal switch + router changed, syncing services in network "default" I0428 15:35:38.506808 3815 services_controller.go:574] Full service sync requested for network=default I0428 15:35:38.529825 3815 iptables.go:108] Creating table: filter chain: FORWARD I0428 15:35:38.537104 3815 iptables.go:108] Creating table: filter chain: OUTPUT I0428 15:35:38.544342 3815 gateway_shared_intf.go:1816] Gateway Creation Complete I0428 15:35:38.544361 3815 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-99.ec2.internal","mac-address":"12:3d:00:67:8a:3b","ip-addresses":["10.0.0.99/24"],"ip-address":"10.0.0.99/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:5ccccf10-2a2a-4556-8e67-bbda603851ad k8s.ovn.org/node-encap-ips:["10.0.0.99"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.99/24"} k8s.ovn.org/zone-name:ip-10-0-0-99.ec2.internal] on node ip-10-0-0-99.ec2.internal I0428 15:35:38.560221 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-99.ec2.internal in network "default" I0428 15:35:38.560271 3815 master.go:548] Adding or Updating local node "ip-10-0-0-99.ec2.internal" for network "default" I0428 15:35:38.560300 3815 node_tracker.go:169] Node ip-10-0-0-99.ec2.internal switch + router changed, syncing services in network "default" I0428 15:35:38.560317 3815 services_controller.go:574] Full service sync requested for network=default I0428 15:35:38.564705 3815 default_node_network_controller.go:519] Node connection status = connected W0428 15:35:38.566162 3815 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-mt5ft: pod openshift-network-diagnostics/network-check-target-mt5ft: no pod IPs found W0428 15:35:38.566183 3815 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-gml9f: pod openshift-multus/network-metrics-daemon-gml9f: no pod IPs found I0428 15:35:38.567375 3815 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-99.ec2.internal for the network default I0428 15:35:38.567617 3815 zone_ic_handler.go:195] Time taken to create transit switch: 224.742µs I0428 15:35:38.606906 3815 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0428 15:35:38.606929 3815 services_controller.go:208] Setting up event handlers for services for network=default I0428 15:35:38.607014 3815 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0428 15:35:38.607068 3815 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0428 15:35:38.607086 3815 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0428 15:35:38.707392 3815 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0428 15:35:38.707414 3815 repair.go:60] Starting repairing loop for services I0428 15:35:38.707507 3815 repair.go:131] Deleted 0 stale service LBs I0428 15:35:38.707531 3815 repair.go:137] Deleted 0 stale Chassis Template Vars I0428 15:35:38.707551 3815 repair.go:62] Finished repairing loop for services: 139.434µs I0428 15:35:38.707582 3815 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0428 15:35:38.707596 3815 services_controller.go:254] Starting workers for network=default I0428 15:35:38.708056 3815 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-mt5ft] creating logical port openshift-network-diagnostics_network-check-target-mt5ft for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:38.708056 3815 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-gml9f] creating logical port openshift-multus_network-metrics-daemon-gml9f for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:38.708293 3815 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-mt5ft I0428 15:35:38.708402 3815 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-gml9f I0428 15:35:38.722647 3815 pod.go:62] [openshift-network-diagnostics/network-check-target-mt5ft] pod update took 14.436386ms I0428 15:35:38.722669 3815 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-mt5ft] addLogicalPort annotation time took 14.464603ms I0428 15:35:38.723656 3815 pods.go:271] [openshift-network-diagnostics/network-check-target-mt5ft] addLogicalPort took 15.613451ms, libovsdb time 740.451µs I0428 15:35:38.726106 3815 pod.go:62] [openshift-multus/network-metrics-daemon-gml9f] pod update took 17.75758ms I0428 15:35:38.726126 3815 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-gml9f] addLogicalPort annotation time took 17.836294ms I0428 15:35:38.727129 3815 pods.go:271] [openshift-multus/network-metrics-daemon-gml9f] addLogicalPort took 19.088016ms, libovsdb time 716.026µs I0428 15:35:38.727692 3815 admin_network_policy_controller.go:227] Starting controller default-network-controller I0428 15:35:38.727718 3815 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0428 15:35:38.727731 3815 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0428 15:35:38.727736 3815 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0428 15:35:38.727926 3815 repair.go:29] Repairing admin network policies took 185.094µs I0428 15:35:38.728026 3815 repair.go:92] Repairing baseline admin network policies took 88.574µs I0428 15:35:38.728548 3815 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0428 15:35:38.728566 3815 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0428 15:35:38.730525 3815 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0428 15:35:38.730545 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0428 15:35:38.730560 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0428 15:35:38.732065 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0428 15:35:38.732084 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0428 15:35:38.732093 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0428 15:35:38.733918 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0428 15:35:38.733937 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0428 15:35:38.733946 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0428 15:35:38.734531 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0428 15:35:38.734556 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0428 15:35:38.734563 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0428 15:35:38.735859 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0428 15:35:38.735880 3815 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0428 15:35:38.735890 3815 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0428 15:35:38.738213 3815 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0428 15:35:38.738230 3815 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0428 15:35:38.738263 3815 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0428 15:35:38.739372 3815 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0428 15:35:38.739390 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0428 15:35:38.739398 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0428 15:35:38.740515 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0428 15:35:38.740539 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0428 15:35:38.740560 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0428 15:35:38.741143 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0428 15:35:38.741161 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0428 15:35:38.741950 3815 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0428 15:35:38.741964 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 15:35:38.742683 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0428 15:35:38.742702 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0428 15:35:38.742727 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 15:35:38.744478 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0428 15:35:38.744499 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0428 15:35:38.744508 3815 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0428 15:35:38.746129 3815 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0428 15:35:38.746149 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0428 15:35:38.746157 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0428 15:35:38.746721 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0428 15:35:38.746741 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0428 15:35:38.746750 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0428 15:35:38.747229 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0428 15:35:38.747265 3815 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0428 15:35:38.747284 3815 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0428 15:35:38.747555 3815 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0428 15:35:38.747573 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0428 15:35:38.747588 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0428 15:35:38.749696 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0428 15:35:38.749714 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0428 15:35:38.749724 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0428 15:35:38.750658 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0428 15:35:38.750678 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0428 15:35:38.750695 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0428 15:35:38.751154 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0428 15:35:38.751169 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0428 15:35:38.751177 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0428 15:35:38.751641 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0428 15:35:38.751661 3815 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0428 15:35:38.751672 3815 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0428 15:35:38.752112 3815 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0428 15:35:38.752131 3815 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0428 15:35:38.752141 3815 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0428 15:35:38.753058 3815 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0428 15:35:38.753076 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0428 15:35:38.753105 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0428 15:35:38.753728 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0428 15:35:38.753747 3815 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0428 15:35:38.753755 3815 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0428 15:35:38.754314 3815 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0428 15:35:38.754332 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0428 15:35:38.754340 3815 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0428 15:35:38.754842 3815 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0428 15:35:38.754866 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0428 15:35:38.754882 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0428 15:35:38.755425 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0428 15:35:38.755441 3815 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0428 15:35:38.755447 3815 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0428 15:35:38.755706 3815 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0428 15:35:38.755727 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0428 15:35:38.755743 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0428 15:35:38.757226 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0428 15:35:38.757260 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0428 15:35:38.757267 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0428 15:35:38.757511 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0428 15:35:38.757530 3815 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0428 15:35:38.757542 3815 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0428 15:35:38.758497 3815 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0428 15:35:38.763370 3815 egressfirewall.go:374] Starting EgressFirewall controller I0428 15:35:38.763404 3815 controller.go:133] Adding controller egress-firewall-controller event handlers I0428 15:35:38.763409 3815 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0428 15:35:38.763427 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0428 15:35:38.763443 3815 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0428 15:35:38.763454 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0428 15:35:38.763465 3815 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0428 15:35:38.763743 3815 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0428 15:35:38.763756 3815 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0428 15:35:38.763762 3815 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0428 15:35:38.763771 3815 egressqos.go:193] Setting up event handlers for EgressQoS I0428 15:35:38.763842 3815 egressfirewall.go:723] Syncing node "ip-10-0-0-99.ec2.internal" for egress firewall I0428 15:35:38.763849 3815 egressqos.go:245] Starting EgressQoS Controller I0428 15:35:38.763852 3815 egressfirewall.go:723] Syncing node "ip-10-0-2-15.ec2.internal" for egress firewall I0428 15:35:38.763856 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0428 15:35:38.763862 3815 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0428 15:35:38.763867 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0428 15:35:38.763871 3815 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0428 15:35:38.763875 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0428 15:35:38.763880 3815 shared_informer.go:356] "Caches are synced" controller="egressqos" I0428 15:35:38.763883 3815 egressqos.go:259] Repairing EgressQoSes I0428 15:35:38.763888 3815 egressqos.go:399] Starting repairing loop for egressqos I0428 15:35:38.763963 3815 egressqos.go:401] Finished repairing loop for egressqos: 75.317µs I0428 15:35:38.763980 3815 egressservice_zone.go:132] Setting up event handlers for Egress Services I0428 15:35:38.764074 3815 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-99.ec2.internal I0428 15:35:38.764132 3815 egressservice_zone.go:210] Starting Egress Services Controller I0428 15:35:38.764142 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 15:35:38.764148 3815 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 15:35:38.764153 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 15:35:38.764159 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 15:35:38.764164 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 15:35:38.764170 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 15:35:38.764175 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0428 15:35:38.764179 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0428 15:35:38.764182 3815 egressservice_zone.go:228] Repairing Egress Services I0428 15:35:38.764181 3815 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-99.ec2.internal : 118.644µs I0428 15:35:38.764460 3815 master_controller.go:88] Starting Admin Policy Based Route Controller I0428 15:35:38.764470 3815 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 15:35:38.764488 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-99.ec2.internal I0428 15:35:38.764502 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-99.ec2.internal: 16.966µs I0428 15:35:38.764511 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-15.ec2.internal I0428 15:35:38.764519 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-15.ec2.internal: 9.013µs I0428 15:35:38.764588 3815 default_network_controller.go:585] Completing all the Watchers took 297.528959ms I0428 15:35:38.764602 3815 default_network_controller.go:589] Starting unidling controllers I0428 15:35:38.764614 3815 unidle.go:46] Registering OVN SB ControllerEvent handler I0428 15:35:38.764623 3815 unidle.go:63] Populating Initial ContollerEvent events I0428 15:35:38.764671 3815 unidle.go:79] Setting up event handlers for services I0428 15:35:38.764769 3815 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0428 15:35:39.064796 3815 default_node_network_controller.go:519] Node connection status = connected I0428 15:35:39.564795 3815 default_node_network_controller.go:519] Node connection status = connected I0428 15:35:40.065315 3815 default_node_network_controller.go:519] Node connection status = connected I0428 15:35:40.566003 3815 default_node_network_controller.go:519] Node connection status = connected I0428 15:35:40.685286 3815 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0428 15:35:41.065503 3815 default_node_network_controller.go:519] Node connection status = connected I0428 15:35:41.085343 3815 bridgeconfig.go:537] Gateway is ready I0428 15:35:41.085366 3815 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0428 15:35:41.099295 3815 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.99. Updating OVN encap IP. I0428 15:35:41.105501 3815 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.99 - it is already configured I0428 15:35:41.105517 3815 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.99/24:{} 172.20.0.1/32:{}]. Updating annotations. I0428 15:35:41.105795 3815 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.99/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-99.ec2.internal","mac-address":"12:3d:00:67:8a:3b","ip-addresses":["10.0.0.99/24"],"ip-address":"10.0.0.99/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:5ccccf10-2a2a-4556-8e67-bbda603851ad k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.99/24"}] on node ip-10-0-0-99.ec2.internal I0428 15:35:41.123181 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-99.ec2.internal in network "default" I0428 15:35:41.123224 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-99.ec2.internal I0428 15:35:41.123226 3815 egressfirewall.go:723] Syncing node "ip-10-0-0-99.ec2.internal" for egress firewall I0428 15:35:41.123187 3815 master.go:548] Adding or Updating local node "ip-10-0-0-99.ec2.internal" for network "default" I0428 15:35:41.123258 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-99.ec2.internal: 37.785µs I0428 15:35:41.123272 3815 node_tracker.go:169] Node ip-10-0-0-99.ec2.internal switch + router changed, syncing services in network "default" I0428 15:35:41.123286 3815 services_controller.go:574] Full service sync requested for network=default I0428 15:35:41.123923 3815 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.0.99 I0428 15:35:41.124445 3815 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-99.ec2.internal I0428 15:35:41.128154 3815 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-99.ec2.internal. Updating no re-route policies I0428 15:35:41.128231 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-r8tzg I0428 15:35:41.128259 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-gms4n I0428 15:35:41.128263 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-z4vsj I0428 15:35:41.128269 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-r8tzg I0428 15:35:41.128273 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-gms4n I0428 15:35:41.128276 3815 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-99.ec2.internal I0428 15:35:41.128280 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-z4vsj I0428 15:35:41.128289 3815 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-99.ec2.internal I0428 15:35:41.128294 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-4bbkv I0428 15:35:41.128298 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-r8tzg after 0 failed attempt(s) I0428 15:35:41.128306 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-z4vsj after 0 failed attempt(s) I0428 15:35:41.128298 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-gms4n after 0 failed attempt(s) I0428 15:35:41.128308 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-gml9f I0428 15:35:41.128308 3815 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-99.ec2.internal after 0 failed attempt(s) I0428 15:35:41.128310 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-4bbkv I0428 15:35:41.128317 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-982gj I0428 15:35:41.128323 3815 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-pf2js I0428 15:35:41.128325 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-99.ec2.internal I0428 15:35:41.128327 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-kwrkp I0428 15:35:41.128333 3815 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-pf2js I0428 15:35:41.128332 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-mt5ft I0428 15:35:41.128330 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-c85c5 I0428 15:35:41.128338 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-kwrkp I0428 15:35:41.128335 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-99.ec2.internal I0428 15:35:41.128342 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-mt5ft I0428 15:35:41.128345 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-c85c5 I0428 15:35:41.128340 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-4bbkv after 0 failed attempt(s) I0428 15:35:41.128350 3815 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-pf2js after 0 failed attempt(s) I0428 15:35:41.128351 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-lg5xq I0428 15:35:41.128359 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-99.ec2.internal after 0 failed attempt(s) I0428 15:35:41.128359 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-kwrkp after 0 failed attempt(s) I0428 15:35:41.128320 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-gml9f I0428 15:35:41.128363 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-lg5xq I0428 15:35:41.128377 3815 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-mt5ft] creating logical port openshift-network-diagnostics_network-check-target-mt5ft for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:41.128395 3815 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-gml9f] creating logical port openshift-multus_network-metrics-daemon-gml9f for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:41.128403 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-lg5xq after 0 failed attempt(s) I0428 15:35:41.128333 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-982gj I0428 15:35:41.128419 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-982gj after 0 failed attempt(s) I0428 15:35:41.128361 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-c85c5 after 0 failed attempt(s) I0428 15:35:41.129165 3815 pods.go:271] [openshift-network-diagnostics/network-check-target-mt5ft] addLogicalPort took 798.155µs, libovsdb time 329.555µs I0428 15:35:41.129182 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-mt5ft after 0 failed attempt(s) I0428 15:35:41.129288 3815 pods.go:271] [openshift-multus/network-metrics-daemon-gml9f] addLogicalPort took 906.246µs, libovsdb time 358.629µs I0428 15:35:41.129310 3815 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-gml9f after 0 failed attempt(s) I0428 15:35:41.139553 3815 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0428 15:35:41.154015 3815 iptables.go:108] Creating table: mangle chain: OUTPUT I0428 15:35:41.157928 3815 iptables.go:108] Creating table: nat chain: OUTPUT I0428 15:35:41.218589 3815 iptables.go:108] Creating table: nat chain: PREROUTING I0428 15:35:41.283799 3815 iptables.go:108] Creating table: filter chain: FORWARD I0428 15:35:41.303190 3815 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0428 15:35:41.303301 3815 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0428 15:35:41.303360 3815 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0428 15:35:41.303431 3815 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0428 15:35:41.303479 3815 gateway_init.go:325] Gateway and management port readiness took 2.743519601s I0428 15:35:41.303507 3815 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0428 15:35:41.303546 3815 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0428 15:35:41.303562 3815 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0428 15:35:41.303574 3815 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 15:35:41.303645 3815 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0428 15:35:41.303727 3815 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 55.291µs I0428 15:35:41.303961 3815 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0428 15:35:41.303986 3815 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.16µs I0428 15:35:41.303994 3815 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 34.237µs I0428 15:35:41.304000 3815 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0428 15:35:41.304010 3815 nad_controller.go:379] [node-nad-controller NAD controller]: started I0428 15:35:41.304018 3815 default_node_network_controller.go:899] Starting the default node network controller I0428 15:35:41.304024 3815 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0428 15:35:41.304037 3815 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.344µs I0428 15:35:41.304183 3815 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0428 15:35:41.304192 3815 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0428 15:35:41.304216 3815 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b59780 0xc0002ec540 0xc0002ec5b0 0xc0002ec620 0xc0002ec690 0xc0002ec700 0xc0002ec770 0xc000b0dd60 0xc0002ec850 0xc0002ec8c0 0xc000cf4280 0xc00645d340 0xc0002ec930 map[0x2ff9080:0xc0028a4ae0 0x2ff95c0:0xc0028a4c60 0x2ffa2e0:0xc0028a4540 0x2ffa580:0xc0028a4a80 0x2ffad60:0xc0028a4b40 0x2ffb000:0xc0028a4ba0 0x2ffb2a0:0xc0028a4c00 0x2ffba80:0xc0028a4cc0 0x2ffbd20:0xc0028a4d20 0x3026980:0xc0010c2900 0x3026d20:0xc0010c2960 0x3027800:0xc000ccac00 0x3029160:0xc001b46000 0x30298a0:0xc0010c2b40 0x302a720:0xc001b46060] 0xc0004df340 0} I0428 15:35:41.304300 3815 gateway.go:151] Starting gateway service sync I0428 15:35:41.305108 3815 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0428 15:35:41.307577 3815 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0428 15:35:41.332261 3815 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0428 15:35:41.333711 3815 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0428 15:35:41.335109 3815 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0428 15:35:41.371314 3815 gateway.go:168] Gateway service sync done. Time taken: 67.001903ms I0428 15:35:41.371626 3815 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32251 I0428 15:35:41.372056 3815 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b59780 0xc0002ec540 0xc0002ec5b0 0xc0002ec620 0xc0002ec690 0xc0002ec700 0xc0002ec770 0xc000b0dd60 0xc0002ec850 0xc0002ec8c0 0xc000cf4280 0xc00645d340 0xc0002ec930 map[0x2ff9080:0xc0028a4ae0 0x2ff95c0:0xc0028a4c60 0x2ffa2e0:0xc0028a4540 0x2ffa580:0xc0028a4a80 0x2ffad60:0xc0028a4b40 0x2ffb000:0xc0028a4ba0 0x2ffb2a0:0xc0028a4c00 0x2ffba80:0xc0028a4cc0 0x2ffbd20:0xc0028a4d20 0x3026980:0xc0010c2900 0x3026d20:0xc0010c2960 0x3027800:0xc000ccac00 0x3029160:0xc001b46000 0x30298a0:0xc0010c2b40 0x302a720:0xc001b46060] 0xc0004df340 0} I0428 15:35:41.372491 3815 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b59780 0xc0002ec540 0xc0002ec5b0 0xc0002ec620 0xc0002ec690 0xc0002ec700 0xc0002ec770 0xc000b0dd60 0xc0002ec850 0xc0002ec8c0 0xc000cf4280 0xc00645d340 0xc0002ec930 map[0x2ff9080:0xc0028a4ae0 0x2ff95c0:0xc0028a4c60 0x2ffa2e0:0xc0028a4540 0x2ffa580:0xc0028a4a80 0x2ffad60:0xc0028a4b40 0x2ffb000:0xc0028a4ba0 0x2ffb2a0:0xc0028a4c00 0x2ffba80:0xc0028a4cc0 0x2ffbd20:0xc0028a4d20 0x3026980:0xc0010c2900 0x3026d20:0xc0010c2960 0x3027800:0xc000ccac00 0x3029160:0xc001b46000 0x30298a0:0xc0010c2b40 0x302a720:0xc001b46060] 0xc0004df340 0} I0428 15:35:41.372529 3815 gateway.go:532] Reconciling gateway with updates I0428 15:35:41.378614 3815 gateway.go:339] Spawning Conntrack Rule Check Thread I0428 15:35:41.378666 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:35:41.378669 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:35:41.378685 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:35:41.378681 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:35:41.378693 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:35:41.378695 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 15:35:41.378698 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:35:41.378705 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 15:35:41.378710 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:35:41.378712 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:35:41.378713 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:35:41.378687 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:35:41.378720 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:35:41.378725 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:35:41.378728 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:35:41.378730 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:35:41.378739 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:35:41.378747 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:35:41.378721 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:35:41.378760 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 15:35:41.378769 3815 default_node_network_controller.go:944] Gateway and management port readiness took 74.561864ms I0428 15:35:41.378776 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 15:35:41.378776 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:35:41.378782 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 15:35:41.378787 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:35:41.378792 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:35:41.378799 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:35:41.378798 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 15:35:41.378800 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 15:35:41.378805 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:35:41.378800 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 15:35:41.378807 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:35:41.378815 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:35:41.378818 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 15:35:41.378701 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:35:41.378819 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:35:41.378823 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 15:35:41.378827 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:35:41.378833 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:35:41.378739 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:35:41.378833 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 15:35:41.378853 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:35:41.378811 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:35:41.378861 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:35:41.378865 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:35:41.378875 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:35:41.378871 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 15:35:41.378871 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:35:41.378883 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:35:41.378886 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 15:35:41.378887 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:35:41.378772 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:35:41.378902 3815 port_claim.go:71] Svc openshift-ingress/router-default: port 31778 is already open, no action needed I0428 15:35:41.378908 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:35:41.378911 3815 port_claim.go:71] Svc openshift-ingress/router-default: port 32510 is already open, no action needed I0428 15:35:41.378918 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:35:41.378781 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:35:41.378941 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:35:41.378952 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:35:41.378854 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:35:41.378960 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:35:41.378962 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:35:41.378749 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:35:41.378972 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:35:41.378976 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:35:41.378978 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:35:41.378984 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 15:35:41.378808 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 15:35:41.378810 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:35:41.379008 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:35:41.379017 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 15:35:41.378814 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:35:41.379028 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:35:41.378820 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:35:41.379121 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:35:41.378825 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:35:41.378824 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 15:35:41.378836 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 15:35:41.378839 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:35:41.378843 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:35:41.378843 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 15:35:41.378837 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 15:35:41.379473 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 15:35:41.378846 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:35:41.378845 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 15:35:41.379614 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 15:35:41.379629 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 15:35:41.378848 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:35:41.379647 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:35:41.378716 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:35:41.379716 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:35:41.378845 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:35:41.379776 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:35:41.378863 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:35:41.378866 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 15:35:41.379870 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 15:35:41.378860 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:35:41.379902 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:35:41.379902 3815 node_ip_handler_linux.go:160] Node IP manager is running I0428 15:35:41.379910 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 15:35:41.378861 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:35:41.379923 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:35:41.378878 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:35:41.379968 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:35:41.378899 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:35:41.378966 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 15:35:41.378967 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 15:35:41.378785 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) W0428 15:35:41.427304 3815 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 15:35:41.427459 3815 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 15:35:41.427599 3815 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0428 15:35:41.436759 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 15:35:41.436788 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 15:35:41.436799 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 15:35:41.436806 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 15:35:41.436813 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 15:35:41.436817 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 15:35:41.436823 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 15:35:41.436828 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 15:35:41.436835 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 15:35:41.479661 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 15:35:41.479669 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 15:35:41.479687 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 15:35:41.479694 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 15:35:41.479703 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 15:35:41.479711 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 15:35:41.523559 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 15:35:41.523588 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 15:35:41.523596 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 15:35:41.523604 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 15:35:41.523611 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 15:35:41.523620 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 15:35:41.523626 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 15:35:41.523653 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 15:35:41.523664 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 15:35:41.523669 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 15:35:41.523675 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 15:35:41.523686 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 15:35:41.523694 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) W0428 15:35:42.326685 3815 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:35:42.932277 3815 default_node_network_controller.go:1380] Starting node controller node sync W0428 15:35:42.932321 3815 egressip_healthcheck.go:76] Health checking using insecure connection I0428 15:35:42.932365 3815 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0428 15:35:42.944829 3815 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 12.530042ms I0428 15:35:42.944932 3815 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-15.ec2.internal", IP: 10.0.2.15 to PMTUD blocking rules I0428 15:35:42.944942 3815 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-15.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:35:42.950886 3815 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0428 15:35:42.951173 3815 egressservice_node.go:93] Setting up event handlers for Egress Services I0428 15:35:42.951426 3815 egressservice_node.go:183] Starting Egress Services Controller I0428 15:35:42.951448 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 15:35:42.951463 3815 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 15:35:42.951469 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 15:35:42.951474 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 15:35:42.951479 3815 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 15:35:42.951483 3815 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 15:35:42.951487 3815 egressservice_node.go:197] Repairing Egress Services I0428 15:35:42.962080 3815 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0428 15:35:42.962095 3815 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 15:35:42.962339 3815 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0428 15:35:42.962351 3815 link_network_manager.go:119] Link manager is running I0428 15:35:42.962359 3815 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0428 15:35:42.962539 3815 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0428 15:35:42.962800 3815 vrf_manager.go:128] VRF manager is running I0428 15:35:42.962933 3815 node_controller_manager.go:437] Removing flows to drop GARP I0428 15:35:42.962947 3815 gateway.go:532] Reconciling gateway with updates I0428 15:35:42.992171 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 15:35:42.992184 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:35:42.992197 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:35:42.992214 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 15:35:42.992217 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:35:42.992217 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:35:42.992231 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:35:42.992233 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:35:42.992267 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:35:42.992233 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:35:42.992274 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 15:35:42.992278 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:35:42.992298 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:35:42.992299 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 15:35:42.992307 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:35:42.992297 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:35:42.992314 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:35:42.992318 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:35:42.992318 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:35:42.992326 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:35:42.992326 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:35:42.992332 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:35:42.992340 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 15:35:42.992343 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:35:42.992338 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:35:42.992346 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:35:42.992349 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:35:42.992359 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:35:42.992362 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:35:42.992370 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:35:42.992373 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 15:35:42.992372 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:35:42.992382 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 15:35:42.992373 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 15:35:42.992388 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:35:42.992397 3815 port_claim.go:71] Svc openshift-ingress/router-default: port 31778 is already open, no action needed I0428 15:35:42.992397 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 15:35:42.992398 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:35:42.992383 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:35:42.992407 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 15:35:42.992405 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:35:42.992411 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:35:42.992416 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:35:42.992416 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:35:42.992420 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:35:42.992418 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 15:35:42.992421 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:35:42.992424 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:35:42.992432 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 15:35:42.992437 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:35:42.992360 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:35:42.992442 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:35:42.992452 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:35:42.992451 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:35:42.992464 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:35:42.992467 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:35:42.992475 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:35:42.992202 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 15:35:42.992484 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:35:42.992269 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:35:42.992300 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:35:42.992624 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 15:35:42.992636 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 15:35:42.992221 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 15:35:42.992647 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 15:35:42.992218 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:35:42.992674 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:35:42.992203 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:35:42.992232 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:35:42.992733 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:35:42.992254 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:35:42.992781 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:35:42.992280 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 15:35:42.992298 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:35:42.992890 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:35:42.992317 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 15:35:42.992317 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:35:42.992922 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:35:42.992323 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 15:35:42.992327 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:35:42.992334 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:35:42.992971 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:35:42.992333 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:35:42.993000 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 15:35:42.992343 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:35:42.993009 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:35:42.992171 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:35:42.993052 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:35:42.993057 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 15:35:42.992347 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:35:42.993062 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:35:42.993065 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 15:35:42.992352 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:35:42.993070 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:35:42.992362 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:35:42.993096 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:35:42.992407 3815 port_claim.go:71] Svc openshift-ingress/router-default: port 32510 is already open, no action needed I0428 15:35:42.992398 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 15:35:42.992406 3815 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:35:42.993266 3815 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:35:42.993272 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 15:35:43.038875 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 15:35:43.038905 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 15:35:43.038915 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 15:35:43.038924 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 15:35:43.038934 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 15:35:43.038943 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 15:35:43.038951 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 15:35:43.038958 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 15:35:43.038967 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 15:35:43.085162 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 15:35:43.085187 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 15:35:43.085195 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 15:35:43.085203 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 15:35:43.085212 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 15:35:43.085220 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 15:35:43.085227 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 15:35:43.085232 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 15:35:43.085250 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 15:35:43.085256 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 15:35:43.085265 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 15:35:43.085274 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 15:35:43.085287 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 15:35:43.085299 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 15:35:43.085304 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 15:35:43.085320 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 15:35:43.085334 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 15:35:43.085341 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 15:35:43.085346 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 15:35:43.085356 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 15:35:43.085367 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 15:35:43.085373 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 15:35:43.130527 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 15:35:43.130535 3815 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 15:35:44.402212 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-99.ec2.internal I0428 15:35:44.402259 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-99.ec2.internal: 52.566µs I0428 15:35:44.429371 3815 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-85884c9ddb-zgwhq] creating logical port openshift-image-registry_image-registry-85884c9ddb-zgwhq for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:44.429519 3815 kube.go:256] Updating pod openshift-image-registry/image-registry-85884c9ddb-zgwhq I0428 15:35:44.449377 3815 pod.go:62] [openshift-image-registry/image-registry-85884c9ddb-zgwhq] pod update took 19.893273ms I0428 15:35:44.449399 3815 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-85884c9ddb-zgwhq] addLogicalPort annotation time took 19.9215ms I0428 15:35:44.450506 3815 pods.go:271] [openshift-image-registry/image-registry-85884c9ddb-zgwhq] addLogicalPort took 21.149085ms, libovsdb time 871.996µs I0428 15:35:44.457369 3815 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-nz4h5] creating logical port openshift-dns_dns-default-nz4h5 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:44.457503 3815 kube.go:256] Updating pod openshift-dns/dns-default-nz4h5 I0428 15:35:44.458924 3815 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-f7pbk] creating logical port openshift-ingress-canary_ingress-canary-f7pbk for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:44.458999 3815 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-f7pbk I0428 15:35:44.472346 3815 pod.go:62] [openshift-dns/dns-default-nz4h5] pod update took 14.885584ms I0428 15:35:44.472377 3815 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-nz4h5] addLogicalPort annotation time took 14.922508ms I0428 15:35:44.473531 3815 pods.go:271] [openshift-dns/dns-default-nz4h5] addLogicalPort took 16.176224ms, libovsdb time 822.631µs I0428 15:35:44.476370 3815 pod.go:62] [openshift-ingress-canary/ingress-canary-f7pbk] pod update took 17.387999ms I0428 15:35:44.476395 3815 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-f7pbk] addLogicalPort annotation time took 17.42006ms I0428 15:35:44.477382 3815 pods.go:271] [openshift-ingress-canary/ingress-canary-f7pbk] addLogicalPort took 18.462897ms, libovsdb time 729.756µs I0428 15:35:44.862324 3815 cni.go:452] [openshift-ingress-canary/ingress-canary-f7pbk 188c56c19c52eb1509d1b1c4f61aaeaed2d0e586881aab0c3fc64fa9e53e3f68 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-f7pbk 188c56c19c52eb1509d1b1c4f61aaeaed2d0e586881aab0c3fc64fa9e53e3f68 network default NAD default NAD key ] I0428 15:35:44.893832 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-f7pbk, hostIfaceName: 188c56c19c52eb1, network: default, NAD default, SandboxID: "188c56c19c52eb1509d1b1c4f61aaeaed2d0e586881aab0c3fc64fa9e53e3f68", PCI device ID: , UID: "14e7c9c2-713b-40c7-9fa4-9451b0afe9bc", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0428 15:35:44.941770 3815 cni.go:473] [openshift-ingress-canary/ingress-canary-f7pbk 188c56c19c52eb1509d1b1c4f61aaeaed2d0e586881aab0c3fc64fa9e53e3f68 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-f7pbk 188c56c19c52eb1509d1b1c4f61aaeaed2d0e586881aab0c3fc64fa9e53e3f68 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:5f:fc:a6:97:6f\",\"name\":\"188c56c19c52eb1\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d77d978a-045a-4469-9571-226c1c7881e4\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:35:46.304828 3815 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-rkswg] creating logical port openshift-insights_insights-operator-6dbb575d6c-rkswg for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:46.304946 3815 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-rkswg I0428 15:35:46.328962 3815 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-rkswg] pod update took 24.044364ms I0428 15:35:46.329006 3815 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-rkswg] addLogicalPort annotation time took 24.091507ms I0428 15:35:46.330087 3815 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-rkswg] addLogicalPort took 25.279116ms, libovsdb time 777.658µs I0428 15:35:46.417773 3815 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-qnn95] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-qnn95 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:46.417899 3815 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-qnn95 I0428 15:35:46.455859 3815 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-qnn95] pod update took 37.998248ms I0428 15:35:46.455923 3815 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-qnn95] addLogicalPort annotation time took 38.066146ms I0428 15:35:46.455951 3815 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-66f5dd849b-gvnkl] creating logical port openshift-ingress_router-default-66f5dd849b-gvnkl for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:46.456096 3815 kube.go:256] Updating pod openshift-ingress/router-default-66f5dd849b-gvnkl I0428 15:35:46.458635 3815 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-qnn95] addLogicalPort took 40.882839ms, libovsdb time 1.757953ms I0428 15:35:46.474343 3815 pod.go:62] [openshift-ingress/router-default-66f5dd849b-gvnkl] pod update took 18.286908ms I0428 15:35:46.474374 3815 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-66f5dd849b-gvnkl] addLogicalPort annotation time took 18.325727ms I0428 15:35:46.475945 3815 pods.go:271] [openshift-ingress/router-default-66f5dd849b-gvnkl] addLogicalPort took 20.008418ms, libovsdb time 1.181554ms I0428 15:35:46.501401 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-qvckz] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-qvckz for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:46.501550 3815 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-qvckz I0428 15:35:46.507476 3815 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-82n8k] creating logical port openshift-console-operator_console-operator-66dbbb6468-82n8k for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:46.507593 3815 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-82n8k I0428 15:35:46.531059 3815 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-qvckz] pod update took 29.550671ms I0428 15:35:46.531083 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-qvckz] addLogicalPort annotation time took 29.580703ms I0428 15:35:46.532545 3815 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-qvckz] addLogicalPort took 31.162929ms, libovsdb time 1.053688ms I0428 15:35:46.533662 3815 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-82n8k] pod update took 26.092209ms I0428 15:35:46.533690 3815 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-82n8k] addLogicalPort annotation time took 26.124588ms I0428 15:35:46.534952 3815 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-82n8k] addLogicalPort took 27.495924ms, libovsdb time 964.563µs I0428 15:35:46.932143 3815 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-rkswg 5175b3cd70eef070a0434c5b8ced6d2dcab9b82678da372ccb3f8b7a4ae7a576 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-rkswg 5175b3cd70eef070a0434c5b8ced6d2dcab9b82678da372ccb3f8b7a4ae7a576 network default NAD default NAD key ] I0428 15:35:46.932408 3815 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-82n8k f6a64585dd6fb80689ef465747e534b46eb7495df40bf2b755f4fc53919f8846 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-82n8k f6a64585dd6fb80689ef465747e534b46eb7495df40bf2b755f4fc53919f8846 network default NAD default NAD key ] I0428 15:35:46.944662 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-82n8k, hostIfaceName: f6a64585dd6fb80, network: default, NAD default, SandboxID: "f6a64585dd6fb80689ef465747e534b46eb7495df40bf2b755f4fc53919f8846", PCI device ID: , UID: "88339728-1c5d-420c-a02f-e0e827ce9242", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0428 15:35:46.944809 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-rkswg, hostIfaceName: 5175b3cd70eef07, network: default, NAD default, SandboxID: "5175b3cd70eef070a0434c5b8ced6d2dcab9b82678da372ccb3f8b7a4ae7a576", PCI device ID: , UID: "87e1d30b-3b0e-4870-b6dd-04ae651ede23", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0428 15:35:46.995950 3815 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-82n8k f6a64585dd6fb80689ef465747e534b46eb7495df40bf2b755f4fc53919f8846 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-82n8k f6a64585dd6fb80689ef465747e534b46eb7495df40bf2b755f4fc53919f8846 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:3d:ee:58:ad:2a\",\"name\":\"f6a64585dd6fb80\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/66ec88f4-d27d-44a6-bf99-3ec2a79a3d4d\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:35:47.002527 3815 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-rkswg 5175b3cd70eef070a0434c5b8ced6d2dcab9b82678da372ccb3f8b7a4ae7a576 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-rkswg 5175b3cd70eef070a0434c5b8ced6d2dcab9b82678da372ccb3f8b7a4ae7a576 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:ee:fa:c6:33:a4\",\"name\":\"5175b3cd70eef07\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/91c6a889-89f7-471e-8502-70321b367aa6\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:35:49.025501 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-18.ec2.internal in network "default" I0428 15:35:49.025530 3815 node_tracker.go:220] Node ip-10-0-1-18.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0428 15:35:49.025561 3815 egressfirewall.go:723] Syncing node "ip-10-0-1-18.ec2.internal" for egress firewall E0428 15:35:49.025587 3815 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-18.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0428 15:35:49.025591 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-18.ec2.internal I0428 15:35:49.025607 3815 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-18.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-18.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-18.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-18.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-18.ec2.internal, error parsing annotation for node ip-10-0-1-18.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] I0428 15:35:49.025610 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-18.ec2.internal: 24.231µs E0428 15:35:49.037378 3815 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-18.ec2.internal annotation: unexpected end of JSON input I0428 15:35:49.037437 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0428 15:35:49.037469 3815 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-18.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0428 15:35:49.054446 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:35:49.054466 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-18.ec2.internal I0428 15:35:49.054484 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-18.ec2.internal: 22.009µs I0428 15:35:49.054467 3815 egressfirewall.go:723] Syncing node "ip-10-0-1-18.ec2.internal" for egress firewall E0428 15:35:49.054465 3815 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-18.ec2.internal annotation: unexpected end of JSON input E0428 15:35:49.054506 3815 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-18.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0428 15:35:49.134593 3815 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-18.ec2.internal annotation: unexpected end of JSON input E0428 15:35:49.134658 3815 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-18.ec2.internal annotation: unexpected end of JSON input I0428 15:35:49.134678 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0428 15:35:49.134727 3815 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-18.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-18.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-18.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-18.ec2.internal" I0428 15:35:49.149726 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0428 15:35:49.319830 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-18.ec2.internal I0428 15:35:49.319837 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:35:49.319852 3815 egressfirewall.go:723] Syncing node "ip-10-0-1-18.ec2.internal" for egress firewall I0428 15:35:49.319857 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-18.ec2.internal: 34.451µs I0428 15:35:49.339595 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:35:49.591711 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:35:52.259450 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0428 15:35:52.268457 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0428 15:35:54.272334 3815 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0428 15:35:54.273185 3815 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 830.826µs I0428 15:35:54.292333 3815 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0428 15:35:55.346321 3815 namespace.go:98] [openshift-service-ca] adding namespace I0428 15:35:55.347223 3815 namespace.go:102] [openshift-service-ca] adding namespace took 876.616µs I0428 15:35:55.366141 3815 namespace.go:142] [openshift-service-ca] updating namespace I0428 15:35:57.029718 3815 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-w82xf] creating logical port openshift-service-ca_service-ca-d9cb48497-w82xf for pod on switch ip-10-0-0-99.ec2.internal I0428 15:35:57.029844 3815 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-w82xf I0428 15:35:57.044836 3815 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-w82xf] pod update took 15.018891ms I0428 15:35:57.044863 3815 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-w82xf] addLogicalPort annotation time took 15.050237ms I0428 15:35:57.046087 3815 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-w82xf] addLogicalPort took 16.384945ms, libovsdb time 922.339µs I0428 15:35:57.444273 3815 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-w82xf 861df81e3eeb3667b9d5be942f05dbbdf136e48e7ae64a3e61efd5781ee75256 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-w82xf 861df81e3eeb3667b9d5be942f05dbbdf136e48e7ae64a3e61efd5781ee75256 network default NAD default NAD key ] I0428 15:35:57.454632 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-w82xf, hostIfaceName: 861df81e3eeb366, network: default, NAD default, SandboxID: "861df81e3eeb3667b9d5be942f05dbbdf136e48e7ae64a3e61efd5781ee75256", PCI device ID: , UID: "e903914a-1060-4852-b418-51ee0a0778bd", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0428 15:35:57.506637 3815 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-w82xf 861df81e3eeb3667b9d5be942f05dbbdf136e48e7ae64a3e61efd5781ee75256 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-w82xf 861df81e3eeb3667b9d5be942f05dbbdf136e48e7ae64a3e61efd5781ee75256 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:f7:8f:85:1b:6d\",\"name\":\"861df81e3eeb366\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/32b8cf87-c681-4f95-b1ec-330fa396b113\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:35:59.132275 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:36:00.948180 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-18.ec2.internal I0428 15:36:00.948212 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-18.ec2.internal: 39.502µs I0428 15:36:00.948263 3815 egressfirewall.go:723] Syncing node "ip-10-0-1-18.ec2.internal" for egress firewall I0428 15:36:00.948319 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0428 15:36:03.854287 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0428 15:36:08.507291 3815 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-1-18.ec2.internal I0428 15:36:08.507316 3815 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-1-18.ec2.internal I0428 15:36:08.507412 3815 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-1-18.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-1-18.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-18.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-18.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-18.ec2.internal I0428 15:36:09.487719 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:36:11.129421 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-hbvmb I0428 15:36:11.129451 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-hbvmb I0428 15:36:11.129430 3815 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-v68c9 I0428 15:36:11.129480 3815 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-v68c9 I0428 15:36:11.129514 3815 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-v68c9, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-v68c9: suppressed error logged: pod openshift-multus/network-metrics-daemon-v68c9: no pod IPs found I0428 15:36:11.129544 3815 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-hbvmb, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-hbvmb: suppressed error logged: pod openshift-network-diagnostics/network-check-target-hbvmb: no pod IPs found I0428 15:36:16.616782 3815 cni.go:452] [openshift-image-registry/image-registry-85884c9ddb-zgwhq f62bc7d08f05fe915190088061ebdd3a4774bea9688b8bd57e0ed871de2abe92 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-85884c9ddb-zgwhq f62bc7d08f05fe915190088061ebdd3a4774bea9688b8bd57e0ed871de2abe92 network default NAD default NAD key ] I0428 15:36:16.626469 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-85884c9ddb-zgwhq, hostIfaceName: f62bc7d08f05fe9, network: default, NAD default, SandboxID: "f62bc7d08f05fe915190088061ebdd3a4774bea9688b8bd57e0ed871de2abe92", PCI device ID: , UID: "4712af51-1f4c-4c34-8e4b-0671f4fb9c8a", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0428 15:36:16.644445 3815 cni.go:452] [openshift-dns/dns-default-nz4h5 e06f9794ea7221f1039479444c2a51c964bc476a97407b44bf2501570db73168 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-nz4h5 e06f9794ea7221f1039479444c2a51c964bc476a97407b44bf2501570db73168 network default NAD default NAD key ] I0428 15:36:16.656190 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-nz4h5, hostIfaceName: e06f9794ea7221f, network: default, NAD default, SandboxID: "e06f9794ea7221f1039479444c2a51c964bc476a97407b44bf2501570db73168", PCI device ID: , UID: "1c2e9ce2-9bd7-47ca-8180-c0d846f18829", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0428 15:36:16.678866 3815 cni.go:473] [openshift-image-registry/image-registry-85884c9ddb-zgwhq f62bc7d08f05fe915190088061ebdd3a4774bea9688b8bd57e0ed871de2abe92 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-85884c9ddb-zgwhq f62bc7d08f05fe915190088061ebdd3a4774bea9688b8bd57e0ed871de2abe92 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:a1:13:17:fa:4c\",\"name\":\"f62bc7d08f05fe9\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fee41a15-7a9b-43da-a14c-06e6a5977e7e\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:16.704350 3815 cni.go:473] [openshift-dns/dns-default-nz4h5 e06f9794ea7221f1039479444c2a51c964bc476a97407b44bf2501570db73168 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-nz4h5 e06f9794ea7221f1039479444c2a51c964bc476a97407b44bf2501570db73168 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:b1:84:32:8f:37\",\"name\":\"e06f9794ea7221f\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/91bb74f6-4639-4d6b-9086-df7712c25f20\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:18.263140 3815 cni.go:452] [openshift-multus/network-metrics-daemon-gml9f 26df069d5573ef790c62a4ee3d782cba15b0a9d37dc8ed8fb9d3359ca91cb3c0 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-gml9f 26df069d5573ef790c62a4ee3d782cba15b0a9d37dc8ed8fb9d3359ca91cb3c0 network default NAD default NAD key ] I0428 15:36:18.263562 3815 cni.go:452] [openshift-network-diagnostics/network-check-target-mt5ft 1ad0ee0401bd8adaec17fb69024d2e49b50e2f36c661e60e422f0212dc70a94f network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-mt5ft 1ad0ee0401bd8adaec17fb69024d2e49b50e2f36c661e60e422f0212dc70a94f network default NAD default NAD key ] I0428 15:36:18.276873 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-gml9f, hostIfaceName: 26df069d5573ef7, network: default, NAD default, SandboxID: "26df069d5573ef790c62a4ee3d782cba15b0a9d37dc8ed8fb9d3359ca91cb3c0", PCI device ID: , UID: "c48fc0e3-112a-406e-9022-cc53909cb18d", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0428 15:36:18.277185 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-mt5ft, hostIfaceName: 1ad0ee0401bd8ad, network: default, NAD default, SandboxID: "1ad0ee0401bd8adaec17fb69024d2e49b50e2f36c661e60e422f0212dc70a94f", PCI device ID: , UID: "a16899d6-87f2-460f-a870-b07c94c7d30e", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0428 15:36:18.329705 3815 cni.go:473] [openshift-network-diagnostics/network-check-target-mt5ft 1ad0ee0401bd8adaec17fb69024d2e49b50e2f36c661e60e422f0212dc70a94f network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-mt5ft 1ad0ee0401bd8adaec17fb69024d2e49b50e2f36c661e60e422f0212dc70a94f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:a2:d1:eb:ef:f1\",\"name\":\"1ad0ee0401bd8ad\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/87e2f322-3d78-4d26-9836-f79ae171324a\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:18.345229 3815 cni.go:473] [openshift-multus/network-metrics-daemon-gml9f 26df069d5573ef790c62a4ee3d782cba15b0a9d37dc8ed8fb9d3359ca91cb3c0 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-gml9f 26df069d5573ef790c62a4ee3d782cba15b0a9d37dc8ed8fb9d3359ca91cb3c0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:c8:80:5e:84:ea\",\"name\":\"26df069d5573ef7\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0dc309a-f5b2-486a-a746-20447a2c098b\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:18.639437 3815 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-qnn95 c8417a90b6f7f38f8eb75d7d9d7a0cfcb55934ab15a3f27503587e6b40755e50 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-qnn95 c8417a90b6f7f38f8eb75d7d9d7a0cfcb55934ab15a3f27503587e6b40755e50 network default NAD default NAD key ] I0428 15:36:18.650287 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-qnn95, hostIfaceName: c8417a90b6f7f38, network: default, NAD default, SandboxID: "c8417a90b6f7f38f8eb75d7d9d7a0cfcb55934ab15a3f27503587e6b40755e50", PCI device ID: , UID: "7c5c0293-12da-4b83-8b14-97bcbedd9bbe", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0428 15:36:18.657192 3815 cni.go:452] [openshift-ingress/router-default-66f5dd849b-gvnkl 53cbd813a4ef46e6d79c15068303c03068a7c3a611191a2a7888b9b7f810b19b network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-66f5dd849b-gvnkl 53cbd813a4ef46e6d79c15068303c03068a7c3a611191a2a7888b9b7f810b19b network default NAD default NAD key ] I0428 15:36:18.668487 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-66f5dd849b-gvnkl, hostIfaceName: 53cbd813a4ef46e, network: default, NAD default, SandboxID: "53cbd813a4ef46e6d79c15068303c03068a7c3a611191a2a7888b9b7f810b19b", PCI device ID: , UID: "6c560bc4-03c1-4786-8055-7cd647644d6e", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0428 15:36:18.693345 3815 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-qvckz e3e0011f2fee136b8c28f9115b307fb924d7b107edc2cde54f4d6d074b54e4d9 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-qvckz e3e0011f2fee136b8c28f9115b307fb924d7b107edc2cde54f4d6d074b54e4d9 network default NAD default NAD key ] I0428 15:36:18.709820 3815 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-qnn95 c8417a90b6f7f38f8eb75d7d9d7a0cfcb55934ab15a3f27503587e6b40755e50 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-qnn95 c8417a90b6f7f38f8eb75d7d9d7a0cfcb55934ab15a3f27503587e6b40755e50 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:6e:f7:9b:7d:6a\",\"name\":\"c8417a90b6f7f38\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c40df6ad-eed0-4e11-8794-4e7f2e39ec11\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:18.721635 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-qvckz, hostIfaceName: e3e0011f2fee136, network: default, NAD default, SandboxID: "e3e0011f2fee136b8c28f9115b307fb924d7b107edc2cde54f4d6d074b54e4d9", PCI device ID: , UID: "85a7bb76-45e3-4836-a291-108422b4ee64", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0428 15:36:18.732076 3815 cni.go:473] [openshift-ingress/router-default-66f5dd849b-gvnkl 53cbd813a4ef46e6d79c15068303c03068a7c3a611191a2a7888b9b7f810b19b network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-66f5dd849b-gvnkl 53cbd813a4ef46e6d79c15068303c03068a7c3a611191a2a7888b9b7f810b19b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:7f:cf:bd:dd:dd\",\"name\":\"53cbd813a4ef46e\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0a7abc64-47d5-46ad-981a-6f0a155996b7\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:18.771835 3815 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-qvckz e3e0011f2fee136b8c28f9115b307fb924d7b107edc2cde54f4d6d074b54e4d9 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-qvckz e3e0011f2fee136b8c28f9115b307fb924d7b107edc2cde54f4d6d074b54e4d9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:0d:91:d7:24:f1\",\"name\":\"e3e0011f2fee136\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e90e412d-077c-4f3b-9711-e2d3377456c0\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:19.869504 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0428 15:36:20.187057 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:36:20.187156 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:36:20.187173 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:36:20.187178 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:36:20.190044 3815 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:20.190062 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:20.190069 3815 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:20.217313 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:36:20.228275 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0428 15:36:20.228299 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0428 15:36:20.238167 3815 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:36:20.249464 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0428 15:36:20.381429 3815 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-6d5f66646f-td77j IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-6d5f66646f-td77j: no pod IPs found I0428 15:36:20.392401 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:36:20.395365 3815 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-h6g4m] creating logical port openshift-insights_insights-runtime-extractor-h6g4m for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:20.395594 3815 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-h6g4m I0428 15:36:20.421405 3815 pod.go:62] [openshift-insights/insights-runtime-extractor-h6g4m] pod update took 25.858795ms I0428 15:36:20.421441 3815 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-h6g4m] addLogicalPort annotation time took 25.901908ms I0428 15:36:20.422977 3815 pods.go:271] [openshift-insights/insights-runtime-extractor-h6g4m] addLogicalPort took 27.633394ms, libovsdb time 1.139769ms I0428 15:36:20.434322 3815 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-fvpf8] creating logical port openshift-console_downloads-5c54f4944f-fvpf8 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:20.434545 3815 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-fvpf8 W0428 15:36:20.437204 3815 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:36:20.444379 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0428 15:36:20.444400 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0428 15:36:20.460206 3815 pod.go:62] [openshift-console/downloads-5c54f4944f-fvpf8] pod update took 25.698075ms I0428 15:36:20.460315 3815 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-fvpf8] addLogicalPort annotation time took 25.810607ms I0428 15:36:20.461713 3815 pods.go:271] [openshift-console/downloads-5c54f4944f-fvpf8] addLogicalPort took 27.412179ms, libovsdb time 994.645µs I0428 15:36:20.536454 3815 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0428 15:36:20.550871 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:36:20.550887 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:36:20.550893 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:36:20.550990 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:20.550998 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:20.551003 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:20.551046 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:20.551054 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:20.551061 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0428 15:36:20.771782 3815 cni.go:452] [openshift-insights/insights-runtime-extractor-h6g4m 8ff45f692e031e47ffe1df5c2a4bed9f5c6e4d360db02ef52d6866570cc22c6e network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-h6g4m 8ff45f692e031e47ffe1df5c2a4bed9f5c6e4d360db02ef52d6866570cc22c6e network default NAD default NAD key ] I0428 15:36:20.785103 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-h6g4m, hostIfaceName: 8ff45f692e031e4, network: default, NAD default, SandboxID: "8ff45f692e031e47ffe1df5c2a4bed9f5c6e4d360db02ef52d6866570cc22c6e", PCI device ID: , UID: "615ca3d8-95bc-4fa4-9acd-36608727bcd8", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0428 15:36:20.802648 3815 cni.go:452] [openshift-console/downloads-5c54f4944f-fvpf8 6b3157d3ad549e64a81f5d65f4ad3812a5bff08dc03e880d176fa449da42560b network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-fvpf8 6b3157d3ad549e64a81f5d65f4ad3812a5bff08dc03e880d176fa449da42560b network default NAD default NAD key ] I0428 15:36:20.813835 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-fvpf8, hostIfaceName: 6b3157d3ad549e6, network: default, NAD default, SandboxID: "6b3157d3ad549e64a81f5d65f4ad3812a5bff08dc03e880d176fa449da42560b", PCI device ID: , UID: "2be84a47-2d16-4786-8e95-addef26e825a", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0428 15:36:20.842285 3815 cni.go:473] [openshift-insights/insights-runtime-extractor-h6g4m 8ff45f692e031e47ffe1df5c2a4bed9f5c6e4d360db02ef52d6866570cc22c6e network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-h6g4m 8ff45f692e031e47ffe1df5c2a4bed9f5c6e4d360db02ef52d6866570cc22c6e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:6d:1a:a2:ea:a4\",\"name\":\"8ff45f692e031e4\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/185a1d6a-1140-476a-9d20-4fef3fa52069\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:20.867586 3815 cni.go:473] [openshift-console/downloads-5c54f4944f-fvpf8 6b3157d3ad549e64a81f5d65f4ad3812a5bff08dc03e880d176fa449da42560b network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-fvpf8 6b3157d3ad549e64a81f5d65f4ad3812a5bff08dc03e880d176fa449da42560b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:bc:9c:94:2a:d7\",\"name\":\"6b3157d3ad549e6\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f6903b77-b5db-453d-9786-164553f26d98\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:20.993365 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-18.ec2.internal in network "default" I0428 15:36:20.993374 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "ip-10-0-1-18.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:36:20.993423 3815 node_tracker.go:235] Node ip-10-0-1-18.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-18.ec2.internal" I0428 15:36:20.993443 3815 node_tracker.go:169] Node ip-10-0-1-18.ec2.internal switch + router changed, syncing services in network "default" I0428 15:36:20.993458 3815 services_controller.go:574] Full service sync requested for network=default W0428 15:36:20.993566 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:36:20.993808 3815 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:36:21.188798 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-18.ec2.internal in network "default" I0428 15:36:21.188833 3815 default_network_controller.go:1025] Node "ip-10-0-1-18.ec2.internal" in remote zone "ip-10-0-1-18.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:36:21.188867 3815 node_tracker.go:169] Node ip-10-0-1-18.ec2.internal switch + router changed, syncing services in network "default" I0428 15:36:21.188883 3815 services_controller.go:574] Full service sync requested for network=default W0428 15:36:21.189300 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:36:21.189551 3815 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:36:21.189964 3815 zone_ic_handler.go:195] Time taken to create transit switch: 307.171µs I0428 15:36:21.189976 3815 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-18.ec2.internal for the network default I0428 15:36:21.191191 3815 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-18.ec2.internal" on network "default" took: 1.547553ms I0428 15:36:21.750429 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-18.ec2.internal I0428 15:36:21.750449 3815 egressfirewall.go:723] Syncing node "ip-10-0-1-18.ec2.internal" for egress firewall I0428 15:36:21.750459 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-18.ec2.internal: 37.786µs I0428 15:36:21.750505 3815 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-18.ec2.internal in network "default" I0428 15:36:21.750550 3815 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-18.ec2.internal", IP: 10.0.1.18 to PMTUD blocking rules I0428 15:36:21.750564 3815 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-18.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:36:21.750568 3815 node_tracker.go:169] Node ip-10-0-1-18.ec2.internal switch + router changed, syncing services in network "default" I0428 15:36:21.750586 3815 services_controller.go:574] Full service sync requested for network=default W0428 15:36:21.750822 3815 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:36:21.751284 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 15:36:21.757157 3815 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-18.ec2.internal. Updating no re-route policies I0428 15:36:21.772870 3815 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-18.ec2.internal", IP: 10.0.1.18 to PMTUD blocking rules I0428 15:36:21.772895 3815 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-18.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:36:22.826524 3815 base_network_controller_pods.go:487] [default/openshift-console/console-59fd5b758b-jgv2c] creating logical port openshift-console_console-59fd5b758b-jgv2c for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:22.826718 3815 kube.go:256] Updating pod openshift-console/console-59fd5b758b-jgv2c I0428 15:36:22.843289 3815 pod.go:62] [openshift-console/console-59fd5b758b-jgv2c] pod update took 16.65985ms I0428 15:36:22.843317 3815 base_network_controller_pods.go:951] [default/openshift-console/console-59fd5b758b-jgv2c] addLogicalPort annotation time took 16.692946ms I0428 15:36:22.844722 3815 pods.go:271] [openshift-console/console-59fd5b758b-jgv2c] addLogicalPort took 18.219786ms, libovsdb time 1.016756ms I0428 15:36:23.061990 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-qzq2h] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-qzq2h for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:23.062122 3815 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-qzq2h I0428 15:36:23.078157 3815 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-qzq2h] pod update took 16.064092ms I0428 15:36:23.078178 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-qzq2h] addLogicalPort annotation time took 16.089245ms I0428 15:36:23.079571 3815 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-qzq2h] addLogicalPort took 17.599435ms, libovsdb time 1.07873ms I0428 15:36:23.189152 3815 cni.go:452] [openshift-console/console-59fd5b758b-jgv2c 89c84e2bf75f0bd0362b8f0f5e9a6b9d6d21bc0c3f0b271869d820ee3e6d1a32 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-59fd5b758b-jgv2c 89c84e2bf75f0bd0362b8f0f5e9a6b9d6d21bc0c3f0b271869d820ee3e6d1a32 network default NAD default NAD key ] I0428 15:36:23.200356 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-59fd5b758b-jgv2c, hostIfaceName: 89c84e2bf75f0bd, network: default, NAD default, SandboxID: "89c84e2bf75f0bd0362b8f0f5e9a6b9d6d21bc0c3f0b271869d820ee3e6d1a32", PCI device ID: , UID: "dba511c5-53af-4326-ad7b-794491bfd8fa", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0428 15:36:23.256622 3815 cni.go:473] [openshift-console/console-59fd5b758b-jgv2c 89c84e2bf75f0bd0362b8f0f5e9a6b9d6d21bc0c3f0b271869d820ee3e6d1a32 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-59fd5b758b-jgv2c 89c84e2bf75f0bd0362b8f0f5e9a6b9d6d21bc0c3f0b271869d820ee3e6d1a32 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:63:d2:a4:5e:18\",\"name\":\"89c84e2bf75f0bd\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bad292d8-4f63-4a91-9c24-84e8345e1e2b\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:36:23.377144 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 15:36:24.195759 3815 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-qzq2h e86b80da4dec9da2768db8d80d13b0844f3df3930bc3d92c6da50a4b3c58ec99 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-qzq2h e86b80da4dec9da2768db8d80d13b0844f3df3930bc3d92c6da50a4b3c58ec99 network default NAD default NAD key ] I0428 15:36:24.205286 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-qzq2h, hostIfaceName: e86b80da4dec9da, network: default, NAD default, SandboxID: "e86b80da4dec9da2768db8d80d13b0844f3df3930bc3d92c6da50a4b3c58ec99", PCI device ID: , UID: "661e78f9-1c5a-43a4-9298-ecc479d889ec", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0428 15:36:24.259877 3815 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-qzq2h e86b80da4dec9da2768db8d80d13b0844f3df3930bc3d92c6da50a4b3c58ec99 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-qzq2h e86b80da4dec9da2768db8d80d13b0844f3df3930bc3d92c6da50a4b3c58ec99 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:da:81:30:58:be\",\"name\":\"e86b80da4dec9da\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f980e0e1-e817-4423-8776-0b9d27627864\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:24.744499 3815 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-18.ec2.internal I0428 15:36:24.744540 3815 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-18.ec2.internal: 51.342µs W0428 15:36:26.133431 3815 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:36:26.396422 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0428 15:36:26.429472 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0428 15:36:27.412069 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 15:36:27.511772 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 15:36:28.428274 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:36:28.428301 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:36:29.370396 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:29.370419 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:29.370425 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:36:32.010292 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 15:36:33.562068 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0428 15:36:33.562114 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 15:36:35.364137 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:35.364262 3815 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0428 15:36:35.406611 3815 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 42.402029ms I0428 15:36:35.406643 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 42.439608ms I0428 15:36:35.408314 3815 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 44.195496ms, libovsdb time 1.215866ms I0428 15:36:35.762985 3815 cni.go:452] [openshift-monitoring/alertmanager-main-1 65d044e1f659ffaac4972207e8e75da22d59bee2ea468335563d6705c97b04bd network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 65d044e1f659ffaac4972207e8e75da22d59bee2ea468335563d6705c97b04bd network default NAD default NAD key ] I0428 15:36:35.773130 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 65d044e1f659ffa, network: default, NAD default, SandboxID: "65d044e1f659ffaac4972207e8e75da22d59bee2ea468335563d6705c97b04bd", PCI device ID: , UID: "665de329-940e-414e-a6fb-12836a69f8f3", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0428 15:36:35.833326 3815 cni.go:473] [openshift-monitoring/alertmanager-main-1 65d044e1f659ffaac4972207e8e75da22d59bee2ea468335563d6705c97b04bd network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 65d044e1f659ffaac4972207e8e75da22d59bee2ea468335563d6705c97b04bd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:05:d4:9c:b6:be\",\"name\":\"65d044e1f659ffa\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6c929dda-f586-4f1c-8150-4100a4a1d762\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:37.040086 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-79fd57d4fd-gbm86] creating logical port openshift-monitoring_thanos-querier-79fd57d4fd-gbm86 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:37.040189 3815 kube.go:256] Updating pod openshift-monitoring/thanos-querier-79fd57d4fd-gbm86 I0428 15:36:37.069060 3815 pod.go:62] [openshift-monitoring/thanos-querier-79fd57d4fd-gbm86] pod update took 28.905694ms I0428 15:36:37.069082 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-79fd57d4fd-gbm86] addLogicalPort annotation time took 28.933359ms I0428 15:36:37.070363 3815 pods.go:271] [openshift-monitoring/thanos-querier-79fd57d4fd-gbm86] addLogicalPort took 30.29797ms, libovsdb time 882.786µs I0428 15:36:37.404273 3815 cni.go:452] [openshift-monitoring/thanos-querier-79fd57d4fd-gbm86 84300af168beac44c5bd44606bdb941dc336383e0d7538e78251fd5cec2bfb7d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-79fd57d4fd-gbm86 84300af168beac44c5bd44606bdb941dc336383e0d7538e78251fd5cec2bfb7d network default NAD default NAD key ] I0428 15:36:37.419310 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-79fd57d4fd-gbm86, hostIfaceName: 84300af168beac4, network: default, NAD default, SandboxID: "84300af168beac44c5bd44606bdb941dc336383e0d7538e78251fd5cec2bfb7d", PCI device ID: , UID: "f431e239-d419-48f7-b501-ba2033ac0177", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] W0428 15:36:37.419642 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:36:37.419788 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 15:36:37.476299 3815 cni.go:473] [openshift-monitoring/thanos-querier-79fd57d4fd-gbm86 84300af168beac44c5bd44606bdb941dc336383e0d7538e78251fd5cec2bfb7d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-79fd57d4fd-gbm86 84300af168beac44c5bd44606bdb941dc336383e0d7538e78251fd5cec2bfb7d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:41:17:be:3e:89\",\"name\":\"84300af168beac4\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9f4868dd-e0ad-42c4-9d3e-014a38e0310b\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:36:37.512759 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:36:37.512783 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:36:37.512788 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:36:38.259628 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:36:38.259648 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:36:38.259652 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 15:36:38.401421 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-df7hr] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-df7hr for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:38.401569 3815 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-df7hr I0428 15:36:38.417693 3815 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-df7hr] pod update took 16.153411ms I0428 15:36:38.417784 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-df7hr] addLogicalPort annotation time took 16.251889ms I0428 15:36:38.419430 3815 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-df7hr] addLogicalPort took 18.025432ms, libovsdb time 1.206508ms I0428 15:36:38.603957 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-867dbdd967-m4lms] creating logical port openshift-monitoring_metrics-server-867dbdd967-m4lms for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:38.604057 3815 kube.go:256] Updating pod openshift-monitoring/metrics-server-867dbdd967-m4lms I0428 15:36:38.621361 3815 pod.go:62] [openshift-monitoring/metrics-server-867dbdd967-m4lms] pod update took 17.334094ms I0428 15:36:38.621385 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-867dbdd967-m4lms] addLogicalPort annotation time took 17.364859ms I0428 15:36:38.622691 3815 pods.go:271] [openshift-monitoring/metrics-server-867dbdd967-m4lms] addLogicalPort took 18.751986ms, libovsdb time 958.279µs I0428 15:36:39.291675 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-9ddb78956-s9fd4] creating logical port openshift-monitoring_telemeter-client-9ddb78956-s9fd4 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:39.291865 3815 kube.go:256] Updating pod openshift-monitoring/telemeter-client-9ddb78956-s9fd4 I0428 15:36:39.296382 3815 cni.go:452] [openshift-monitoring/metrics-server-867dbdd967-m4lms 03f6d997cdbfd06e15bd265f1d12144401f69bf95b2b02dd7c3adf394b59aee1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-867dbdd967-m4lms 03f6d997cdbfd06e15bd265f1d12144401f69bf95b2b02dd7c3adf394b59aee1 network default NAD default NAD key ] I0428 15:36:39.306870 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-867dbdd967-m4lms, hostIfaceName: 03f6d997cdbfd06, network: default, NAD default, SandboxID: "03f6d997cdbfd06e15bd265f1d12144401f69bf95b2b02dd7c3adf394b59aee1", PCI device ID: , UID: "93cbb6a5-202a-42ad-b82b-949ecb17d541", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0428 15:36:39.311462 3815 pod.go:62] [openshift-monitoring/telemeter-client-9ddb78956-s9fd4] pod update took 19.625289ms I0428 15:36:39.311489 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-9ddb78956-s9fd4] addLogicalPort annotation time took 19.659831ms I0428 15:36:39.312766 3815 pods.go:271] [openshift-monitoring/telemeter-client-9ddb78956-s9fd4] addLogicalPort took 21.116191ms, libovsdb time 931.309µs I0428 15:36:39.361817 3815 cni.go:473] [openshift-monitoring/metrics-server-867dbdd967-m4lms 03f6d997cdbfd06e15bd265f1d12144401f69bf95b2b02dd7c3adf394b59aee1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-867dbdd967-m4lms 03f6d997cdbfd06e15bd265f1d12144401f69bf95b2b02dd7c3adf394b59aee1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:a9:20:02:a9:20\",\"name\":\"03f6d997cdbfd06\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/320be2a5-3c79-4ef6-ab4f-12bbdbe7554e\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:39.369314 3815 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-df7hr ee7fdae5a0e0c05223a255c32f4d0824b0717b22b582ec1bae78842af90a5479 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-df7hr ee7fdae5a0e0c05223a255c32f4d0824b0717b22b582ec1bae78842af90a5479 network default NAD default NAD key ] I0428 15:36:39.380188 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-df7hr, hostIfaceName: ee7fdae5a0e0c05, network: default, NAD default, SandboxID: "ee7fdae5a0e0c05223a255c32f4d0824b0717b22b582ec1bae78842af90a5479", PCI device ID: , UID: "e868a9c3-e900-4699-965c-7b8c4d974a74", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0428 15:36:39.441203 3815 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-df7hr ee7fdae5a0e0c05223a255c32f4d0824b0717b22b582ec1bae78842af90a5479 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-df7hr ee7fdae5a0e0c05223a255c32f4d0824b0717b22b582ec1bae78842af90a5479 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:a1:7d:5d:08:75\",\"name\":\"ee7fdae5a0e0c05\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8705307b-50a6-4fc5-b340-f6ac63a7f83c\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:39.659372 3815 cni.go:452] [openshift-monitoring/telemeter-client-9ddb78956-s9fd4 39ee06d83857ebaaa1586c141a6f9c89a2b6d5610213805bc8635bbe7519cfd9 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-9ddb78956-s9fd4 39ee06d83857ebaaa1586c141a6f9c89a2b6d5610213805bc8635bbe7519cfd9 network default NAD default NAD key ] I0428 15:36:39.670158 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-9ddb78956-s9fd4, hostIfaceName: 39ee06d83857eba, network: default, NAD default, SandboxID: "39ee06d83857ebaaa1586c141a6f9c89a2b6d5610213805bc8635bbe7519cfd9", PCI device ID: , UID: "435c9018-3d5a-4b53-8fd5-9fa437833e0c", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0428 15:36:39.727108 3815 cni.go:473] [openshift-monitoring/telemeter-client-9ddb78956-s9fd4 39ee06d83857ebaaa1586c141a6f9c89a2b6d5610213805bc8635bbe7519cfd9 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-9ddb78956-s9fd4 39ee06d83857ebaaa1586c141a6f9c89a2b6d5610213805bc8635bbe7519cfd9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:a9:cd:7f:28:23\",\"name\":\"39ee06d83857eba\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/190c6270-3299-4317-ab51-850adceb96a0\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:40.322836 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:40.322977 3815 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0428 15:36:40.396946 3815 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 74.006057ms I0428 15:36:40.396978 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 74.042537ms I0428 15:36:40.398645 3815 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 75.835256ms, libovsdb time 1.192856ms W0428 15:36:41.277138 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:36:41.277162 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:36:41.287211 3815 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-7d88f6558c-2wvlr IP address from the namespace address-set, err: pod openshift-console/console-7d88f6558c-2wvlr: no pod IPs found W0428 15:36:43.253281 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0428 15:36:43.332534 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 15:36:43.332554 3815 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 15:36:43.332560 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 15:36:43.332563 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier I0428 15:36:46.921013 3815 cni.go:452] [openshift-image-registry/image-registry-85884c9ddb-zgwhq f62bc7d08f05fe915190088061ebdd3a4774bea9688b8bd57e0ed871de2abe92 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-85884c9ddb-zgwhq f62bc7d08f05fe915190088061ebdd3a4774bea9688b8bd57e0ed871de2abe92 network default NAD default NAD key ] I0428 15:36:47.012014 3815 cni.go:473] [openshift-image-registry/image-registry-85884c9ddb-zgwhq f62bc7d08f05fe915190088061ebdd3a4774bea9688b8bd57e0ed871de2abe92 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-85884c9ddb-zgwhq f62bc7d08f05fe915190088061ebdd3a4774bea9688b8bd57e0ed871de2abe92 network default NAD default NAD key default], result "{}", err I0428 15:36:47.326747 3815 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85884c9ddb-zgwhq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:36:47.326780 3815 pods.go:176] Deleting pod: openshift-image-registry/image-registry-85884c9ddb-zgwhq I0428 15:36:47.328311 3815 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-85884c9ddb-zgwhq, ips: 10.129.0.5 I0428 15:36:47.328346 3815 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85884c9ddb-zgwhq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:36:47.338735 3815 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7b68f5bd6b-tqq27] creating logical port openshift-image-registry_image-registry-7b68f5bd6b-tqq27 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:47.338889 3815 kube.go:256] Updating pod openshift-image-registry/image-registry-7b68f5bd6b-tqq27 I0428 15:36:47.360657 3815 pod.go:62] [openshift-image-registry/image-registry-7b68f5bd6b-tqq27] pod update took 21.812202ms I0428 15:36:47.360681 3815 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7b68f5bd6b-tqq27] addLogicalPort annotation time took 21.844165ms I0428 15:36:47.362035 3815 pods.go:271] [openshift-image-registry/image-registry-7b68f5bd6b-tqq27] addLogicalPort took 23.318734ms, libovsdb time 961.584µs I0428 15:36:47.514043 3815 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85884c9ddb-zgwhq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:36:47.514070 3815 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85884c9ddb-zgwhq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:36:47.517808 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-85884c9ddb-zgwhq I0428 15:36:47.517836 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-85884c9ddb-zgwhq W0428 15:36:47.597350 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0428 15:36:48.249286 3815 cni.go:452] [openshift-monitoring/prometheus-k8s-1 f0134353c65c226149852f50d1c0935b478ea19329701cc5c438d54190d120b6 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 f0134353c65c226149852f50d1c0935b478ea19329701cc5c438d54190d120b6 network default NAD default NAD key ] I0428 15:36:48.260797 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: f0134353c65c226, network: default, NAD default, SandboxID: "f0134353c65c226149852f50d1c0935b478ea19329701cc5c438d54190d120b6", PCI device ID: , UID: "d9a4529b-0d8e-4156-9c6e-d95ef7d5d2de", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0428 15:36:48.265349 3815 cni.go:452] [openshift-image-registry/image-registry-7b68f5bd6b-tqq27 46ccdfa4ee7b34ce284bcf4735a13815d8dc4212056483f6a1cf9495accdd23c network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7b68f5bd6b-tqq27 46ccdfa4ee7b34ce284bcf4735a13815d8dc4212056483f6a1cf9495accdd23c network default NAD default NAD key ] I0428 15:36:48.278721 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7b68f5bd6b-tqq27, hostIfaceName: 46ccdfa4ee7b34c, network: default, NAD default, SandboxID: "46ccdfa4ee7b34ce284bcf4735a13815d8dc4212056483f6a1cf9495accdd23c", PCI device ID: , UID: "1df2ef94-ccbf-4e29-be7a-4d05fca02ac7", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0428 15:36:48.329175 3815 cni.go:473] [openshift-monitoring/prometheus-k8s-1 f0134353c65c226149852f50d1c0935b478ea19329701cc5c438d54190d120b6 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 f0134353c65c226149852f50d1c0935b478ea19329701cc5c438d54190d120b6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:46:ec:e7:95:62\",\"name\":\"f0134353c65c226\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5aedbe2e-6cff-453a-ae79-6326dea8dd7a\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:36:48.359638 3815 cni.go:473] [openshift-image-registry/image-registry-7b68f5bd6b-tqq27 46ccdfa4ee7b34ce284bcf4735a13815d8dc4212056483f6a1cf9495accdd23c network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7b68f5bd6b-tqq27 46ccdfa4ee7b34ce284bcf4735a13815d8dc4212056483f6a1cf9495accdd23c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:24:a0:a9:41:d8\",\"name\":\"46ccdfa4ee7b34c\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5d4987f1-ff78-4fa3-9eff-4b3caaf7cfd6\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:36:48.612809 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:36:48.612835 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:36:48.728928 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:36:48.823306 3815 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-684d65d899-8rldf IP address from the namespace address-set, err: pod openshift-console/console-684d65d899-8rldf: no pod IPs found W0428 15:36:48.837356 3815 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-684d65d899-729tn IP address from the namespace address-set, err: pod openshift-console/console-684d65d899-729tn: no pod IPs found W0428 15:36:54.392315 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:36:55.585054 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 15:36:55.585077 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 15:36:55.585083 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 15:36:55.585088 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier I0428 15:36:57.189210 3815 cni.go:452] [openshift-console/console-59fd5b758b-jgv2c 89c84e2bf75f0bd0362b8f0f5e9a6b9d6d21bc0c3f0b271869d820ee3e6d1a32 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-59fd5b758b-jgv2c 89c84e2bf75f0bd0362b8f0f5e9a6b9d6d21bc0c3f0b271869d820ee3e6d1a32 network default NAD default NAD key ] I0428 15:36:57.294254 3815 cni.go:473] [openshift-console/console-59fd5b758b-jgv2c 89c84e2bf75f0bd0362b8f0f5e9a6b9d6d21bc0c3f0b271869d820ee3e6d1a32 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-59fd5b758b-jgv2c 89c84e2bf75f0bd0362b8f0f5e9a6b9d6d21bc0c3f0b271869d820ee3e6d1a32 network default NAD default NAD key default], result "{}", err I0428 15:36:57.374857 3815 obj_retry.go:498] Detected object openshift-console/console-59fd5b758b-jgv2c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:36:57.374901 3815 pods.go:176] Deleting pod: openshift-console/console-59fd5b758b-jgv2c I0428 15:36:57.376232 3815 pods.go:236] Attempting to release IPs for pod: openshift-console/console-59fd5b758b-jgv2c, ips: 10.129.0.16 I0428 15:36:57.376273 3815 obj_retry.go:498] Detected object openshift-console/console-59fd5b758b-jgv2c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:36:57.385068 3815 base_network_controller_pods.go:487] [default/openshift-console/console-bb57d8f65-gvnpf] creating logical port openshift-console_console-bb57d8f65-gvnpf for pod on switch ip-10-0-0-99.ec2.internal I0428 15:36:57.385189 3815 kube.go:256] Updating pod openshift-console/console-bb57d8f65-gvnpf I0428 15:36:57.401876 3815 pod.go:62] [openshift-console/console-bb57d8f65-gvnpf] pod update took 16.728379ms I0428 15:36:57.401903 3815 base_network_controller_pods.go:951] [default/openshift-console/console-bb57d8f65-gvnpf] addLogicalPort annotation time took 16.76436ms I0428 15:36:57.403370 3815 pods.go:271] [openshift-console/console-bb57d8f65-gvnpf] addLogicalPort took 18.357966ms, libovsdb time 1.043642ms I0428 15:36:57.604606 3815 obj_retry.go:492] Detected object openshift-console/console-59fd5b758b-jgv2c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:36:57.604632 3815 obj_retry.go:492] Detected object openshift-console/console-59fd5b758b-jgv2c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:36:57.611718 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-59fd5b758b-jgv2c I0428 15:36:57.611748 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-59fd5b758b-jgv2c I0428 15:36:57.798598 3815 cni.go:452] [openshift-console/console-bb57d8f65-gvnpf a32d30fe44a4ee0c592cb9db06b582469ffb4d9f3ec5fc8f1b88720ac1ac6ba5 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-bb57d8f65-gvnpf a32d30fe44a4ee0c592cb9db06b582469ffb4d9f3ec5fc8f1b88720ac1ac6ba5 network default NAD default NAD key ] I0428 15:36:57.808220 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-bb57d8f65-gvnpf, hostIfaceName: a32d30fe44a4ee0, network: default, NAD default, SandboxID: "a32d30fe44a4ee0c592cb9db06b582469ffb4d9f3ec5fc8f1b88720ac1ac6ba5", PCI device ID: , UID: "6fbdf1b8-224a-4b0d-bc4c-d6bae8ae51dd", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0428 15:36:57.861411 3815 cni.go:473] [openshift-console/console-bb57d8f65-gvnpf a32d30fe44a4ee0c592cb9db06b582469ffb4d9f3ec5fc8f1b88720ac1ac6ba5 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-bb57d8f65-gvnpf a32d30fe44a4ee0c592cb9db06b582469ffb4d9f3ec5fc8f1b88720ac1ac6ba5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:f9:bb:a9:54:b1\",\"name\":\"a32d30fe44a4ee0\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a710d6c3-2732-401a-9e58-45a51ef6bbcf\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:37:04.489172 3815 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0drvvxv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:04.489777 3815 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0drvvxv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:04.489811 3815 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0drvvxv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:37:04.489823 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0428 15:37:04.489836 3815 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0drvvxv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:37:04.489842 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0428 15:37:05.539065 3815 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0drvvxv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:05.539094 3815 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0drvvxv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:05.539114 3815 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0drvvxv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:05.539127 3815 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0drvvxv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:37:05.717623 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:37:05.717647 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:37:05.717653 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:37:05.717664 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:37:05.717669 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:37:05.717673 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:37:05.788802 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:37:05.788830 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:37:05.788837 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 15:37:06.564459 3815 obj_retry.go:498] Detected object openshift-console/console-7d88f6558c-blsj7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:06.565063 3815 obj_retry.go:498] Detected object openshift-console/console-7d88f6558c-blsj7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:07.394159 3815 obj_retry.go:492] Detected object openshift-console/console-7d88f6558c-blsj7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:07.394182 3815 obj_retry.go:492] Detected object openshift-console/console-7d88f6558c-blsj7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:07.401782 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d88f6558c-blsj7 I0428 15:37:07.401804 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d88f6558c-blsj7 I0428 15:37:08.588523 3815 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0428 15:37:08.602509 3815 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0428 15:37:08.614008 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:37:08.614034 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:37:08.683476 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0428 15:37:09.577375 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:37:09.608902 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:37:11.683595 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0428 15:37:11.683606 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0428 15:37:11.683666 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0428 15:37:12.542896 3815 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85884c9ddb-wwswf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:12.543568 3815 obj_retry.go:498] Detected object openshift-image-registry/image-registry-85884c9ddb-wwswf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0428 15:37:12.709316 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0428 15:37:12.709406 3815 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85884c9ddb-wwswf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:12.709426 3815 obj_retry.go:492] Detected object openshift-image-registry/image-registry-85884c9ddb-wwswf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:12.721379 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-85884c9ddb-wwswf I0428 15:37:12.721403 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-85884c9ddb-wwswf I0428 15:37:13.822915 3815 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0428 15:37:23.739462 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 15:37:29.765535 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:34.058638 3815 obj_retry.go:498] Detected object openshift-console/console-59fd5b758b-cd72k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:34.059235 3815 obj_retry.go:498] Detected object openshift-console/console-59fd5b758b-cd72k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:34.779581 3815 obj_retry.go:492] Detected object openshift-console/console-59fd5b758b-cd72k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:34.779605 3815 obj_retry.go:492] Detected object openshift-console/console-59fd5b758b-cd72k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:34.787948 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-59fd5b758b-cd72k I0428 15:37:34.787980 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-59fd5b758b-cd72k I0428 15:37:36.395860 3815 obj_retry.go:498] Detected object openshift-ingress/router-default-6cdf88fc49-5p48k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:36.396481 3815 obj_retry.go:498] Detected object openshift-ingress/router-default-6cdf88fc49-5p48k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:36.790602 3815 obj_retry.go:492] Detected object openshift-ingress/router-default-6cdf88fc49-5p48k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:36.790624 3815 obj_retry.go:492] Detected object openshift-ingress/router-default-6cdf88fc49-5p48k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:36.798321 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-6cdf88fc49-5p48k I0428 15:37:36.798343 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-6cdf88fc49-5p48k W0428 15:37:41.864316 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:37:41.864365 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0428 15:37:55.739627 3815 cni.go:452] [openshift-monitoring/alertmanager-main-1 65d044e1f659ffaac4972207e8e75da22d59bee2ea468335563d6705c97b04bd network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 65d044e1f659ffaac4972207e8e75da22d59bee2ea468335563d6705c97b04bd network default NAD default NAD key ] I0428 15:37:55.816283 3815 cni.go:473] [openshift-monitoring/alertmanager-main-1 65d044e1f659ffaac4972207e8e75da22d59bee2ea468335563d6705c97b04bd network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 65d044e1f659ffaac4972207e8e75da22d59bee2ea468335563d6705c97b04bd network default NAD default NAD key default], result "{}", err I0428 15:37:55.877819 3815 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:55.877923 3815 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0428 15:37:55.879275 3815 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.129.0.18 I0428 15:37:55.879309 3815 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:37:56.183088 3815 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:56.183116 3815 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:37:56.190893 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0428 15:37:56.190918 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0428 15:37:56.234193 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:37:56.234360 3815 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0428 15:37:56.254936 3815 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 20.648727ms I0428 15:37:56.254956 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 20.672299ms I0428 15:37:56.256319 3815 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 22.140901ms, libovsdb time 1.015595ms I0428 15:37:56.597554 3815 cni.go:452] [openshift-monitoring/alertmanager-main-1 5ca051862d450f84e743590669409896b62b61a540524a10038ca907bd7e421b network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 5ca051862d450f84e743590669409896b62b61a540524a10038ca907bd7e421b network default NAD default NAD key ] I0428 15:37:56.607031 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 5ca051862d450f8, network: default, NAD default, SandboxID: "5ca051862d450f84e743590669409896b62b61a540524a10038ca907bd7e421b", PCI device ID: , UID: "de7707a6-10ad-4f3a-91b6-dadb201ee5e6", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0428 15:37:56.660022 3815 cni.go:473] [openshift-monitoring/alertmanager-main-1 5ca051862d450f84e743590669409896b62b61a540524a10038ca907bd7e421b network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 5ca051862d450f84e743590669409896b62b61a540524a10038ca907bd7e421b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:41:69:e8:2c:66\",\"name\":\"5ca051862d450f8\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ac11ac6f-2397-44b4-b48c-5ff101621487\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:00.017359 3815 cni.go:452] [openshift-monitoring/prometheus-k8s-1 f0134353c65c226149852f50d1c0935b478ea19329701cc5c438d54190d120b6 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 f0134353c65c226149852f50d1c0935b478ea19329701cc5c438d54190d120b6 network default NAD default NAD key ] I0428 15:38:00.100856 3815 cni.go:473] [openshift-monitoring/prometheus-k8s-1 f0134353c65c226149852f50d1c0935b478ea19329701cc5c438d54190d120b6 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 f0134353c65c226149852f50d1c0935b478ea19329701cc5c438d54190d120b6 network default NAD default NAD key default], result "{}", err I0428 15:38:00.166379 3815 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:00.166406 3815 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0428 15:38:00.167801 3815 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.129.0.23 I0428 15:38:00.167830 3815 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:00.929924 3815 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:00.929950 3815 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:00.937458 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0428 15:38:00.937490 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0428 15:38:00.980848 3815 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:38:00.980954 3815 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0428 15:38:01.013270 3815 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 32.32558ms I0428 15:38:01.013292 3815 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 32.382982ms I0428 15:38:01.014495 3815 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 33.667735ms, libovsdb time 882.649µs I0428 15:38:01.343578 3815 cni.go:452] [openshift-monitoring/prometheus-k8s-1 2f78ca436cf6e50a765eaf29670177f41d3c7c29a3d6bdfc3c88f49df2250903 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 2f78ca436cf6e50a765eaf29670177f41d3c7c29a3d6bdfc3c88f49df2250903 network default NAD default NAD key ] I0428 15:38:01.353513 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 2f78ca436cf6e50, network: default, NAD default, SandboxID: "2f78ca436cf6e50a765eaf29670177f41d3c7c29a3d6bdfc3c88f49df2250903", PCI device ID: , UID: "6ecefa1d-357d-42b3-9057-1a6b42d51cca", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0428 15:38:01.405609 3815 cni.go:473] [openshift-monitoring/prometheus-k8s-1 2f78ca436cf6e50a765eaf29670177f41d3c7c29a3d6bdfc3c88f49df2250903 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 2f78ca436cf6e50a765eaf29670177f41d3c7c29a3d6bdfc3c88f49df2250903 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:86:65:66:e7:1d\",\"name\":\"2f78ca436cf6e50\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fa48acb4-c17e-4104-9e69-467e050c1464\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:38:16.650475 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:16.650497 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:16.650504 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 15:38:17.967940 3815 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:17.968615 3815 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:18.672470 3815 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:18.672494 3815 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:18.680892 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0428 15:38:18.680922 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0428 15:38:19.079122 3815 obj_retry.go:498] Detected object openshift-console/console-bb57d8f65-mj62v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:19.079699 3815 obj_retry.go:498] Detected object openshift-console/console-bb57d8f65-mj62v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:19.757654 3815 obj_retry.go:492] Detected object openshift-console/console-bb57d8f65-mj62v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:19.757682 3815 obj_retry.go:492] Detected object openshift-console/console-bb57d8f65-mj62v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:19.767799 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-bb57d8f65-mj62v I0428 15:38:19.767823 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-bb57d8f65-mj62v I0428 15:38:30.105406 3815 cni.go:452] [openshift-console/console-bb57d8f65-gvnpf a32d30fe44a4ee0c592cb9db06b582469ffb4d9f3ec5fc8f1b88720ac1ac6ba5 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-bb57d8f65-gvnpf a32d30fe44a4ee0c592cb9db06b582469ffb4d9f3ec5fc8f1b88720ac1ac6ba5 network default NAD default NAD key ] I0428 15:38:30.188977 3815 cni.go:473] [openshift-console/console-bb57d8f65-gvnpf a32d30fe44a4ee0c592cb9db06b582469ffb4d9f3ec5fc8f1b88720ac1ac6ba5 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-bb57d8f65-gvnpf a32d30fe44a4ee0c592cb9db06b582469ffb4d9f3ec5fc8f1b88720ac1ac6ba5 network default NAD default NAD key default], result "{}", err I0428 15:38:30.236187 3815 obj_retry.go:498] Detected object openshift-console/console-bb57d8f65-gvnpf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:30.236223 3815 pods.go:176] Deleting pod: openshift-console/console-bb57d8f65-gvnpf I0428 15:38:30.237468 3815 pods.go:236] Attempting to release IPs for pod: openshift-console/console-bb57d8f65-gvnpf, ips: 10.129.0.25 I0428 15:38:30.237497 3815 obj_retry.go:498] Detected object openshift-console/console-bb57d8f65-gvnpf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.058907 3815 obj_retry.go:492] Detected object openshift-console/console-bb57d8f65-gvnpf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:31.058930 3815 obj_retry.go:492] Detected object openshift-console/console-bb57d8f65-gvnpf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:31.066632 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-bb57d8f65-gvnpf I0428 15:38:31.066659 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-bb57d8f65-gvnpf W0428 15:39:02.230521 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:39:02.230546 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0428 15:39:03.530372 3815 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:03.530954 3815 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:03.870473 3815 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:03.870495 3815 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:03.880869 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0428 15:39:03.880902 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0428 15:40:05.450367 3815 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-tpcf7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:05.450934 3815 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-tpcf7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:06.358031 3815 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-tpcf7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:06.358049 3815 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-tpcf7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:21.657329 3815 namespace.go:98] [tsf] adding namespace I0428 15:41:21.658279 3815 namespace.go:102] [tsf] adding namespace took 923.544µs I0428 15:41:21.677513 3815 namespace.go:142] [tsf] updating namespace I0428 15:41:21.696479 3815 namespace.go:142] [tsf] updating namespace I0428 15:41:26.431306 3815 namespace.go:98] [tsf-keycloak] adding namespace I0428 15:41:26.431449 3815 namespace.go:98] [tsf-tpa] adding namespace I0428 15:41:26.431554 3815 namespace.go:98] [cert-manager-operator] adding namespace I0428 15:41:26.431647 3815 namespace.go:98] [tsf-tas] adding namespace I0428 15:41:26.432252 3815 namespace.go:102] [tsf-keycloak] adding namespace took 911.576µs I0428 15:41:26.432924 3815 namespace.go:102] [tsf-tpa] adding namespace took 1.459112ms I0428 15:41:26.433586 3815 namespace.go:102] [cert-manager-operator] adding namespace took 2.016808ms I0428 15:41:26.434128 3815 namespace.go:102] [tsf-tas] adding namespace took 2.466522ms I0428 15:41:26.439522 3815 namespace.go:98] [rhbk-operator] adding namespace I0428 15:41:26.439576 3815 namespace.go:98] [konflux-ui] adding namespace I0428 15:41:26.439618 3815 namespace.go:98] [rhtpa-operator] adding namespace I0428 15:41:26.440143 3815 namespace.go:102] [rhbk-operator] adding namespace took 608.211µs I0428 15:41:26.440673 3815 namespace.go:102] [konflux-ui] adding namespace took 1.084253ms I0428 15:41:26.440699 3815 namespace.go:98] [konflux-operator] adding namespace I0428 15:41:26.441135 3815 namespace.go:102] [konflux-operator] adding namespace took 429.433µs I0428 15:41:26.441609 3815 namespace.go:102] [rhtpa-operator] adding namespace took 1.97581ms I0428 15:41:26.454015 3815 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:41:26.467138 3815 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:41:26.485516 3815 namespace.go:142] [tsf-tpa] updating namespace I0428 15:41:26.485560 3815 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:41:26.488301 3815 namespace.go:142] [tsf-tas] updating namespace I0428 15:41:26.488444 3815 namespace.go:142] [rhbk-operator] updating namespace I0428 15:41:26.496664 3815 namespace.go:142] [tsf-tpa] updating namespace I0428 15:41:26.502025 3815 namespace.go:142] [konflux-ui] updating namespace I0428 15:41:26.502080 3815 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:41:26.502233 3815 namespace.go:142] [konflux-operator] updating namespace I0428 15:41:26.506051 3815 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:41:26.562162 3815 namespace.go:142] [tsf-tas] updating namespace I0428 15:41:26.583320 3815 namespace.go:142] [tsf-tas] updating namespace I0428 15:41:26.586908 3815 namespace.go:142] [rhbk-operator] updating namespace I0428 15:41:26.610003 3815 namespace.go:142] [rhbk-operator] updating namespace I0428 15:41:26.618428 3815 namespace.go:142] [konflux-ui] updating namespace I0428 15:41:26.633346 3815 namespace.go:142] [konflux-ui] updating namespace I0428 15:41:26.640415 3815 namespace.go:142] [tsf-tpa] updating namespace I0428 15:41:26.644392 3815 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:41:26.662161 3815 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:41:26.685791 3815 namespace.go:142] [konflux-operator] updating namespace I0428 15:41:26.710365 3815 namespace.go:142] [konflux-operator] updating namespace I0428 15:41:26.710541 3815 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:41:26.728613 3815 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:41:27.682992 3815 namespace.go:142] [tsf-tpa] updating namespace I0428 15:41:27.694859 3815 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:41:35.620617 3815 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbtxj4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:35.621198 3815 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbtxj4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:35.621233 3815 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbtxj4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:35.621260 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:41:35.621270 3815 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbtxj4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:35.621274 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:41:35.644488 3815 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tlw7z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:35.644994 3815 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tlw7z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:35.645018 3815 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tlw7z of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:35.645026 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:41:35.645033 3815 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tlw7z of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:35.645040 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:41:36.646324 3815 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbtxj4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:36.646348 3815 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbtxj4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:36.646373 3815 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbtxj4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:36.646392 3815 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dbtxj4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:36.675948 3815 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tlw7z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:36.675972 3815 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tlw7z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:36.675983 3815 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tlw7z of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:36.675991 3815 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tlw7z of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:37.633957 3815 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01587h5t8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:37.634501 3815 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01587h5t8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:37.634532 3815 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01587h5t8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:37.634542 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:41:37.634552 3815 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01587h5t8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:37.634556 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:41:38.649387 3815 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf22k4fm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:38.649928 3815 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf22k4fm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:38.649953 3815 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf22k4fm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:38.649961 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:41:38.649972 3815 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf22k4fm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:38.649976 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:41:38.666039 3815 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01587h5t8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:38.666065 3815 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01587h5t8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:38.666087 3815 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01587h5t8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:38.666097 3815 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01587h5t8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:39.681327 3815 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf22k4fm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:39.681348 3815 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf22k4fm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:39.681359 3815 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf22k4fm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:39.681368 3815 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf22k4fm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:41.667319 3815 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gdl9j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:41.667934 3815 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gdl9j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:41.667968 3815 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gdl9j of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:41.667982 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:41:41.667993 3815 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gdl9j of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:41.667998 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:41:42.673933 3815 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kw5xp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:42.674471 3815 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kw5xp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:41:42.674497 3815 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kw5xp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:42.674508 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:41:42.674518 3815 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kw5xp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:41:42.674522 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:41:42.695388 3815 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gdl9j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:42.695408 3815 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gdl9j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:42.695421 3815 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gdl9j of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:42.695431 3815 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136gdl9j of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:43.707296 3815 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kw5xp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:43.707319 3815 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kw5xp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:43.707337 3815 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kw5xp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:41:43.707348 3815 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7kw5xp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:41:45.691158 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0428 15:41:46.023474 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0428 15:41:46.037787 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0428 15:41:51.192759 3815 namespace.go:98] [cert-manager] adding namespace I0428 15:41:51.193589 3815 namespace.go:102] [cert-manager] adding namespace took 803.853µs I0428 15:41:51.236369 3815 namespace.go:142] [cert-manager] updating namespace I0428 15:41:51.254943 3815 namespace.go:142] [cert-manager] updating namespace W0428 15:41:56.842255 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 15:41:56.842273 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0428 15:41:59.168928 3815 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0428 15:42:01.733057 3815 namespace.go:142] [openshift-operators] updating namespace I0428 15:42:01.752215 3815 namespace.go:142] [openshift-operators] updating namespace I0428 15:42:02.674283 3815 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-zht8t] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-zht8t for pod on switch ip-10-0-0-99.ec2.internal I0428 15:42:02.674427 3815 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-zht8t I0428 15:42:02.693297 3815 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-zht8t] pod update took 18.897698ms I0428 15:42:02.693325 3815 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-zht8t] addLogicalPort annotation time took 18.931238ms I0428 15:42:02.694461 3815 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-zht8t] addLogicalPort took 20.200549ms, libovsdb time 817.291µs I0428 15:42:03.032707 3815 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-zht8t c4bc2235175f56e33432eaad36fbff86ae183a70b9df47956b3977a257b14069 network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-zht8t c4bc2235175f56e33432eaad36fbff86ae183a70b9df47956b3977a257b14069 network default NAD default NAD key ] I0428 15:42:03.041936 3815 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-zht8t, hostIfaceName: c4bc2235175f56e, network: default, NAD default, SandboxID: "c4bc2235175f56e33432eaad36fbff86ae183a70b9df47956b3977a257b14069", PCI device ID: , UID: "86807668-09ce-44bd-90a2-49f427cbc713", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0428 15:42:03.095136 3815 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-zht8t c4bc2235175f56e33432eaad36fbff86ae183a70b9df47956b3977a257b14069 network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-zht8t c4bc2235175f56e33432eaad36fbff86ae183a70b9df47956b3977a257b14069 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:be:1e:58:8d:fc\",\"name\":\"c4bc2235175f56e\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/459d36ea-ec6b-41ed-8045-adb3dc5de416\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:42:03.959200 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 W0428 15:42:05.819639 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0428 15:42:10.765352 3815 namespace.go:98] [trusted-artifact-signer] adding namespace I0428 15:42:10.766324 3815 namespace.go:102] [trusted-artifact-signer] adding namespace took 943.116µs I0428 15:42:10.831377 3815 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:42:10.858949 3815 namespace.go:142] [trusted-artifact-signer] updating namespace W0428 15:42:10.878698 3815 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0428 15:42:12.160642 3815 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59c77684cc-qjl5q] creating logical port cert-manager_cert-manager-59c77684cc-qjl5q for pod on switch ip-10-0-0-99.ec2.internal I0428 15:42:12.161056 3815 kube.go:256] Updating pod cert-manager/cert-manager-59c77684cc-qjl5q I0428 15:42:12.181319 3815 pod.go:62] [cert-manager/cert-manager-59c77684cc-qjl5q] pod update took 20.301302ms I0428 15:42:12.181342 3815 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-59c77684cc-qjl5q] addLogicalPort annotation time took 20.333411ms I0428 15:42:12.182520 3815 pods.go:271] [cert-manager/cert-manager-59c77684cc-qjl5q] addLogicalPort took 21.899049ms, libovsdb time 842.115µs I0428 15:42:12.546370 3815 cni.go:452] [cert-manager/cert-manager-59c77684cc-qjl5q afcfcc3415bbe003ec7901e2e8cd7516dba7451785201dc16cc68561a43a1dff network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-59c77684cc-qjl5q afcfcc3415bbe003ec7901e2e8cd7516dba7451785201dc16cc68561a43a1dff network default NAD default NAD key ] I0428 15:42:12.556378 3815 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59c77684cc-qjl5q, hostIfaceName: afcfcc3415bbe00, network: default, NAD default, SandboxID: "afcfcc3415bbe003ec7901e2e8cd7516dba7451785201dc16cc68561a43a1dff", PCI device ID: , UID: "2afb2f85-3e78-4b89-a22d-b35ae11b2f41", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0428 15:42:12.610409 3815 cni.go:473] [cert-manager/cert-manager-59c77684cc-qjl5q afcfcc3415bbe003ec7901e2e8cd7516dba7451785201dc16cc68561a43a1dff network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-59c77684cc-qjl5q afcfcc3415bbe003ec7901e2e8cd7516dba7451785201dc16cc68561a43a1dff network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:5f:b8:45:99:69\",\"name\":\"afcfcc3415bbe00\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b9af3324-493c-4e29-a49f-0e007fa94339\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:42:13.148126 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0428 15:42:14.292943 3815 namespace.go:142] [openshift-operators] updating namespace I0428 15:42:14.741771 3815 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-2k6gj] creating logical port openshift-operators_openshift-pipelines-operator-6bdf6c8794-2k6gj for pod on switch ip-10-0-0-99.ec2.internal I0428 15:42:14.741930 3815 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-2k6gj I0428 15:42:14.766679 3815 pod.go:62] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-2k6gj] pod update took 24.776426ms I0428 15:42:14.766705 3815 base_network_controller_pods.go:951] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-2k6gj] addLogicalPort annotation time took 24.805607ms I0428 15:42:14.768112 3815 pods.go:271] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-2k6gj] addLogicalPort took 26.35747ms, libovsdb time 1.032644ms I0428 15:42:15.590877 3815 cni.go:452] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-2k6gj f30bbc5514b53d170511a2d0d48d41c9844eeb6c2fa852323ae97ab6f758f0b6 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-2k6gj f30bbc5514b53d170511a2d0d48d41c9844eeb6c2fa852323ae97ab6f758f0b6 network default NAD default NAD key ] I0428 15:42:15.601203 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6bdf6c8794-2k6gj, hostIfaceName: f30bbc5514b53d1, network: default, NAD default, SandboxID: "f30bbc5514b53d170511a2d0d48d41c9844eeb6c2fa852323ae97ab6f758f0b6", PCI device ID: , UID: "321e3872-c62c-4b2e-859b-41ebf459a52e", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0428 15:42:15.656821 3815 cni.go:473] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-2k6gj f30bbc5514b53d170511a2d0d48d41c9844eeb6c2fa852323ae97ab6f758f0b6 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-2k6gj f30bbc5514b53d170511a2d0d48d41c9844eeb6c2fa852323ae97ab6f758f0b6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:3e:07:aa:7e:3b\",\"name\":\"f30bbc5514b53d1\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/848f93d1-4a66-428a-9aab-685b06a89154\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:42:16.760115 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:42:24.962942 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=60 I0428 15:42:27.812456 3815 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0428 15:42:27.812513 3815 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-zm9xv IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-zm9xv: no pod IPs found I0428 15:42:30.849988 3815 namespace.go:98] [openshift-pipelines] adding namespace I0428 15:42:30.850824 3815 namespace.go:102] [openshift-pipelines] adding namespace took 811.463µs I0428 15:42:30.867800 3815 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:42:31.581053 3815 namespace.go:142] [cert-manager] updating namespace I0428 15:42:31.781739 3815 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:42:32.043107 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0428 15:42:32.183444 3815 namespace.go:142] [konflux-operator] updating namespace I0428 15:42:32.386342 3815 namespace.go:142] [konflux-ui] updating namespace I0428 15:42:32.585204 3815 namespace.go:142] [open-cluster-management-2pv84gmg8k1purftr303td5tpat9d92k] updating namespace I0428 15:42:32.986256 3815 namespace.go:142] [rhbk-operator] updating namespace I0428 15:42:33.186720 3815 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:42:33.389918 3815 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:42:33.590266 3815 namespace.go:142] [tsf] updating namespace I0428 15:42:33.789404 3815 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:42:33.990309 3815 namespace.go:142] [tsf-tas] updating namespace I0428 15:42:34.193452 3815 namespace.go:142] [tsf-tpa] updating namespace I0428 15:42:34.340098 3815 namespace.go:142] [cert-manager] updating namespace I0428 15:42:34.357820 3815 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:42:34.371009 3815 namespace.go:142] [default] updating namespace I0428 15:42:34.387884 3815 namespace.go:142] [konflux-operator] updating namespace I0428 15:42:34.399380 3815 namespace.go:142] [konflux-ui] updating namespace I0428 15:42:34.418447 3815 namespace.go:142] [open-cluster-management-2pv84gmg8k1purftr303td5tpat9d92k] updating namespace I0428 15:42:34.426225 3815 namespace.go:142] [openshift] updating namespace I0428 15:42:34.440104 3815 namespace.go:142] [rhbk-operator] updating namespace I0428 15:42:34.457035 3815 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:42:34.471201 3815 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:42:34.482479 3815 namespace.go:142] [tsf] updating namespace I0428 15:42:34.496011 3815 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:42:34.515156 3815 namespace.go:142] [tsf-tas] updating namespace I0428 15:42:34.535275 3815 namespace.go:142] [tsf-tpa] updating namespace I0428 15:42:34.582913 3815 namespace.go:142] [cert-manager] updating namespace I0428 15:42:34.686635 3815 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:42:34.795723 3815 namespace.go:142] [default] updating namespace I0428 15:42:34.895123 3815 namespace.go:142] [konflux-operator] updating namespace I0428 15:42:34.985442 3815 namespace.go:142] [konflux-ui] updating namespace I0428 15:42:35.079191 3815 namespace.go:142] [open-cluster-management-2pv84gmg8k1purftr303td5tpat9d92k] updating namespace I0428 15:42:35.176972 3815 namespace.go:142] [openshift] updating namespace I0428 15:42:35.275679 3815 namespace.go:142] [rhbk-operator] updating namespace I0428 15:42:35.379694 3815 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:42:35.477525 3815 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:42:35.575486 3815 namespace.go:142] [tsf] updating namespace I0428 15:42:35.681349 3815 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:42:35.781436 3815 namespace.go:142] [tsf-tas] updating namespace I0428 15:42:35.884457 3815 namespace.go:142] [tsf-tpa] updating namespace I0428 15:42:36.175421 3815 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:42:41.105448 3815 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:42:41.105987 3815 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:42:41.134687 3815 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:42:41.134705 3815 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:42:41.141308 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0428 15:42:41.141329 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0428 15:42:43.858746 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs for pod on switch ip-10-0-0-99.ec2.internal I0428 15:42:43.858877 3815 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs I0428 15:42:43.873756 3815 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs] pod update took 14.933823ms I0428 15:42:43.873776 3815 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs] addLogicalPort annotation time took 14.958902ms I0428 15:42:43.875113 3815 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs] addLogicalPort took 16.382752ms, libovsdb time 806.344µs I0428 15:42:44.228660 3815 cni.go:452] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs 586cc9240e2fedf99f8f008f9646f04c6fb63ef29ae5fdd05861cfbc374a2f3a network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs 586cc9240e2fedf99f8f008f9646f04c6fb63ef29ae5fdd05861cfbc374a2f3a network default NAD default NAD key ] I0428 15:42:44.237915 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs, hostIfaceName: 586cc9240e2fedf, network: default, NAD default, SandboxID: "586cc9240e2fedf99f8f008f9646f04c6fb63ef29ae5fdd05861cfbc374a2f3a", PCI device ID: , UID: "24e7a5c0-befb-414c-a680-13d435b29740", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0428 15:42:44.292562 3815 cni.go:473] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs 586cc9240e2fedf99f8f008f9646f04c6fb63ef29ae5fdd05861cfbc374a2f3a network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-kccjs 586cc9240e2fedf99f8f008f9646f04c6fb63ef29ae5fdd05861cfbc374a2f3a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:9e:f7:e1:c6:fc\",\"name\":\"586cc9240e2fedf\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cfec636c-d84c-49ea-9e55-d95e5836b371\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:42:47.926284 3815 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:42:47.926301 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:42:47.926306 3815 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:42:52.047208 3815 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0428 15:42:52.095583 3815 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:42:52.095600 3815 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:42:52.095604 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:42:52.158801 3815 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0428 15:42:52.173335 3815 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:42:52.173350 3815 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:42:52.173356 3815 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:42:52.173361 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0428 15:42:56.958852 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=94 I0428 15:43:14.176745 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 W0428 15:43:15.132413 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0428 15:43:15.308756 3815 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:43:15.309314 3815 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:43:15.338972 3815 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:43:15.338990 3815 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:43:15.346169 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0428 15:43:15.346191 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0428 15:43:17.392499 3815 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0428 15:43:17.393099 3815 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0428 15:43:17.393118 3815 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0428 15:43:17.395803 3815 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0428 15:43:27.960381 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-55fb698565-d9pnb] creating logical port openshift-pipelines_tekton-chains-controller-55fb698565-d9pnb for pod on switch ip-10-0-0-99.ec2.internal I0428 15:43:27.960473 3815 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-55fb698565-d9pnb I0428 15:43:27.978337 3815 pod.go:62] [openshift-pipelines/tekton-chains-controller-55fb698565-d9pnb] pod update took 17.883395ms I0428 15:43:27.978372 3815 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-chains-controller-55fb698565-d9pnb] addLogicalPort annotation time took 17.926548ms I0428 15:43:27.979946 3815 pods.go:271] [openshift-pipelines/tekton-chains-controller-55fb698565-d9pnb] addLogicalPort took 19.541928ms, libovsdb time 1.04766ms I0428 15:43:28.333794 3815 cni.go:452] [openshift-pipelines/tekton-chains-controller-55fb698565-d9pnb cc2b3821d331818884fd3037b69339ef5feb212d7e3916d0aef730453dd12014 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-d9pnb cc2b3821d331818884fd3037b69339ef5feb212d7e3916d0aef730453dd12014 network default NAD default NAD key ] I0428 15:43:28.343973 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-55fb698565-d9pnb, hostIfaceName: cc2b3821d331818, network: default, NAD default, SandboxID: "cc2b3821d331818884fd3037b69339ef5feb212d7e3916d0aef730453dd12014", PCI device ID: , UID: "2142ab13-59f1-4e0a-be9c-f89ffc641b3e", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0428 15:43:28.397113 3815 cni.go:473] [openshift-pipelines/tekton-chains-controller-55fb698565-d9pnb cc2b3821d331818884fd3037b69339ef5feb212d7e3916d0aef730453dd12014 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-d9pnb cc2b3821d331818884fd3037b69339ef5feb212d7e3916d0aef730453dd12014 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:f3:7d:4f:52:dc\",\"name\":\"cc2b3821d331818\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/28fa90ff-c084-4f28-8b4b-3dd92b1e1eec\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:43:32.254050 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 15:43:32.254070 3815 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 15:43:32.255095 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0428 15:43:32.255110 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0428 15:43:33.152346 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0428 15:43:33.262499 3815 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 15:43:35.951129 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=195 I0428 15:43:46.637877 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-64c5c8474c-7k5gr] creating logical port openshift-pipelines_tekton-results-api-64c5c8474c-7k5gr for pod on switch ip-10-0-0-99.ec2.internal I0428 15:43:46.638003 3815 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-64c5c8474c-7k5gr I0428 15:43:46.654322 3815 pod.go:62] [openshift-pipelines/tekton-results-api-64c5c8474c-7k5gr] pod update took 16.358999ms I0428 15:43:46.654349 3815 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-api-64c5c8474c-7k5gr] addLogicalPort annotation time took 16.390833ms I0428 15:43:46.655817 3815 pods.go:271] [openshift-pipelines/tekton-results-api-64c5c8474c-7k5gr] addLogicalPort took 17.964136ms, libovsdb time 994.645µs I0428 15:43:46.728040 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-24tvq] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-585694b54-24tvq for pod on switch ip-10-0-0-99.ec2.internal I0428 15:43:46.728163 3815 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-585694b54-24tvq I0428 15:43:46.742388 3815 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-24tvq] pod update took 14.25007ms I0428 15:43:46.742434 3815 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-24tvq] addLogicalPort annotation time took 14.298665ms I0428 15:43:46.743675 3815 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-24tvq] addLogicalPort took 15.651573ms, libovsdb time 879.474µs I0428 15:43:47.600203 3815 cni.go:452] [openshift-pipelines/tekton-results-api-64c5c8474c-7k5gr b7f24cfbe5aa29475843fd7fdc3d33329158464b6cfddd9db7dc2be234179f8f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-7k5gr b7f24cfbe5aa29475843fd7fdc3d33329158464b6cfddd9db7dc2be234179f8f network default NAD default NAD key ] I0428 15:43:47.609862 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-64c5c8474c-7k5gr, hostIfaceName: b7f24cfbe5aa294, network: default, NAD default, SandboxID: "b7f24cfbe5aa29475843fd7fdc3d33329158464b6cfddd9db7dc2be234179f8f", PCI device ID: , UID: "c90bdcf0-0679-4e2f-85a2-a8e0b8d98169", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0428 15:43:47.663770 3815 cni.go:473] [openshift-pipelines/tekton-results-api-64c5c8474c-7k5gr b7f24cfbe5aa29475843fd7fdc3d33329158464b6cfddd9db7dc2be234179f8f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-7k5gr b7f24cfbe5aa29475843fd7fdc3d33329158464b6cfddd9db7dc2be234179f8f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:c5:e7:28:32:ba\",\"name\":\"b7f24cfbe5aa294\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4340452e-f066-4716-b531-6090388de216\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:43:47.690536 3815 cni.go:452] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-24tvq 212032175b7ef4fb29fcda96d8752a527e8032ea9b9c20a6a451d76a94d7cb84 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-24tvq 212032175b7ef4fb29fcda96d8752a527e8032ea9b9c20a6a451d76a94d7cb84 network default NAD default NAD key ] I0428 15:43:47.700888 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-585694b54-24tvq, hostIfaceName: 212032175b7ef4f, network: default, NAD default, SandboxID: "212032175b7ef4fb29fcda96d8752a527e8032ea9b9c20a6a451d76a94d7cb84", PCI device ID: , UID: "7ff7b899-fbe2-46be-8397-02cc0efae12f", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0428 15:43:47.754984 3815 cni.go:473] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-24tvq 212032175b7ef4fb29fcda96d8752a527e8032ea9b9c20a6a451d76a94d7cb84 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-24tvq 212032175b7ef4fb29fcda96d8752a527e8032ea9b9c20a6a451d76a94d7cb84 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:3e:dc:60:9f:45\",\"name\":\"212032175b7ef4f\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e1ed3581-ebcc-4eea-954a-4cba2923db41\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:43:52.441355 3815 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:43:52.441378 3815 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:43:52.441386 3815 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:43:58.483324 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:43:58.483343 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:43:58.483349 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0428 15:44:04.575610 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-cfzbp] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-cfzbp for pod on switch ip-10-0-0-99.ec2.internal I0428 15:44:04.575723 3815 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-cfzbp I0428 15:44:04.588376 3815 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-cfzbp] pod update took 12.699192ms I0428 15:44:04.588400 3815 base_network_controller_pods.go:951] [default/openshift-pipelines/tkn-cli-serve-784b566968-cfzbp] addLogicalPort annotation time took 12.729277ms I0428 15:44:04.589746 3815 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-cfzbp] addLogicalPort took 14.151351ms, libovsdb time 878.328µs I0428 15:44:04.964107 3815 cni.go:452] [openshift-pipelines/tkn-cli-serve-784b566968-cfzbp 22ba30074eed084adf694434e8cffd3ac76de4e23d474e9500bd0642b6e8124d network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-cfzbp 22ba30074eed084adf694434e8cffd3ac76de4e23d474e9500bd0642b6e8124d network default NAD default NAD key ] I0428 15:44:04.974007 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-cfzbp, hostIfaceName: 22ba30074eed084, network: default, NAD default, SandboxID: "22ba30074eed084adf694434e8cffd3ac76de4e23d474e9500bd0642b6e8124d", PCI device ID: , UID: "8a12ea19-27af-411a-b342-053026146f87", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0428 15:44:05.026997 3815 cni.go:473] [openshift-pipelines/tkn-cli-serve-784b566968-cfzbp 22ba30074eed084adf694434e8cffd3ac76de4e23d474e9500bd0642b6e8124d network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-cfzbp 22ba30074eed084adf694434e8cffd3ac76de4e23d474e9500bd0642b6e8124d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:31:61:4a:b8:d5\",\"name\":\"22ba30074eed084\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d48137f7-b8e3-4df5-b805-315e3d76ecff\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:44:10.627601 3815 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:44:10.628272 3815 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:44:10.669971 3815 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:44:10.669993 3815 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:44:10.680562 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0428 15:44:10.680586 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0428 15:44:13.054692 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0428 15:44:13.248642 3815 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-c69598c95-4rgtl] creating logical port tsf-tas_fulcio-server-c69598c95-4rgtl for pod on switch ip-10-0-0-99.ec2.internal I0428 15:44:13.248763 3815 kube.go:256] Updating pod tsf-tas/fulcio-server-c69598c95-4rgtl I0428 15:44:13.306760 3815 pod.go:62] [tsf-tas/fulcio-server-c69598c95-4rgtl] pod update took 58.01401ms I0428 15:44:13.306781 3815 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-c69598c95-4rgtl] addLogicalPort annotation time took 58.047509ms I0428 15:44:13.307960 3815 pods.go:271] [tsf-tas/fulcio-server-c69598c95-4rgtl] addLogicalPort took 59.339526ms, libovsdb time 830.8µs I0428 15:44:13.425950 3815 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-xpjqx-hkq2d] creating logical port tsf-tas_ctlog-createtree-job-xpjqx-hkq2d for pod on switch ip-10-0-0-99.ec2.internal I0428 15:44:13.426074 3815 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-xpjqx-hkq2d I0428 15:44:13.485680 3815 pod.go:62] [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d] pod update took 59.630541ms I0428 15:44:13.485700 3815 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-createtree-job-xpjqx-hkq2d] addLogicalPort annotation time took 59.65661ms I0428 15:44:13.487101 3815 pods.go:271] [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d] addLogicalPort took 61.17005ms, libovsdb time 933.491µs I0428 15:44:13.612325 3815 cni.go:452] [tsf-tas/fulcio-server-c69598c95-4rgtl e94d23371bd099c1eb810e1372f6135d4e57786d7217c9e145a356d255e5acac network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-c69598c95-4rgtl e94d23371bd099c1eb810e1372f6135d4e57786d7217c9e145a356d255e5acac network default NAD default NAD key ] I0428 15:44:13.622375 3815 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-c69598c95-4rgtl, hostIfaceName: e94d23371bd099c, network: default, NAD default, SandboxID: "e94d23371bd099c1eb810e1372f6135d4e57786d7217c9e145a356d255e5acac", PCI device ID: , UID: "bdacf494-9e0c-4ecd-8c90-d8b9fd91d46f", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0428 15:44:13.682673 3815 cni.go:473] [tsf-tas/fulcio-server-c69598c95-4rgtl e94d23371bd099c1eb810e1372f6135d4e57786d7217c9e145a356d255e5acac network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-c69598c95-4rgtl e94d23371bd099c1eb810e1372f6135d4e57786d7217c9e145a356d255e5acac network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:91:96:8d:ca:89\",\"name\":\"e94d23371bd099c\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/74c53bb7-c17e-42c6-8acd-1def9a2c2fd9\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:44:13.800752 3815 cni.go:452] [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d 7cfa1bfef90b953775637f8c4cac1fb1788f4f89fb5ff16b981d3f29825fb806 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d 7cfa1bfef90b953775637f8c4cac1fb1788f4f89fb5ff16b981d3f29825fb806 network default NAD default NAD key ] I0428 15:44:13.810974 3815 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-xpjqx-hkq2d, hostIfaceName: 7cfa1bfef90b953, network: default, NAD default, SandboxID: "7cfa1bfef90b953775637f8c4cac1fb1788f4f89fb5ff16b981d3f29825fb806", PCI device ID: , UID: "f613b9ca-2d10-4516-bc01-5f2534fdf1ae", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0428 15:44:13.864393 3815 cni.go:473] [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d 7cfa1bfef90b953775637f8c4cac1fb1788f4f89fb5ff16b981d3f29825fb806 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d 7cfa1bfef90b953775637f8c4cac1fb1788f4f89fb5ff16b981d3f29825fb806 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:98:1c:cc:19:2b\",\"name\":\"7cfa1bfef90b953\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0da5f5f8-7944-4e09-b1ac-1b2d4edf0ab5\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:44:13.935075 3815 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-6w4l5-548k2] creating logical port tsf-tas_rekor-createtree-job-6w4l5-548k2 for pod on switch ip-10-0-0-99.ec2.internal I0428 15:44:13.935184 3815 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-6w4l5-548k2 I0428 15:44:13.957888 3815 pod.go:62] [tsf-tas/rekor-createtree-job-6w4l5-548k2] pod update took 22.736722ms I0428 15:44:13.957908 3815 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-6w4l5-548k2] addLogicalPort annotation time took 22.762569ms I0428 15:44:13.958975 3815 pods.go:271] [tsf-tas/rekor-createtree-job-6w4l5-548k2] addLogicalPort took 23.918997ms, libovsdb time 725.93µs I0428 15:44:14.298814 3815 cni.go:452] [tsf-tas/rekor-createtree-job-6w4l5-548k2 824ea037ae92b8ac89ad13aee8b131ee5178d9dee902b137b9bd010c15653d54 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-6w4l5-548k2 824ea037ae92b8ac89ad13aee8b131ee5178d9dee902b137b9bd010c15653d54 network default NAD default NAD key ] I0428 15:44:14.308805 3815 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-6w4l5-548k2, hostIfaceName: 824ea037ae92b8a, network: default, NAD default, SandboxID: "824ea037ae92b8ac89ad13aee8b131ee5178d9dee902b137b9bd010c15653d54", PCI device ID: , UID: "5efd262f-d334-4d4a-b118-80e9a01d29d8", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0428 15:44:14.362467 3815 cni.go:473] [tsf-tas/rekor-createtree-job-6w4l5-548k2 824ea037ae92b8ac89ad13aee8b131ee5178d9dee902b137b9bd010c15653d54 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-6w4l5-548k2 824ea037ae92b8ac89ad13aee8b131ee5178d9dee902b137b9bd010c15653d54 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:95:36:1f:e5:18\",\"name\":\"824ea037ae92b8a\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/981aaeab-4b56-4620-b353-e2e90d274788\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:44:15.570311 3815 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0428 15:44:15.570334 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0428 15:44:15.597195 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0428 15:44:15.597214 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server I0428 15:44:16.661837 3815 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-2bcc8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:44:16.662406 3815 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-2bcc8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:44:17.561040 3815 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-2bcc8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:44:17.561063 3815 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-2bcc8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:44:23.567934 3815 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0428 15:44:23.567953 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0428 15:44:27.536154 3815 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0428 15:44:27.556016 3815 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0428 15:44:29.597234 3815 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0428 15:44:30.676801 3815 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-wx9lc-82lcb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:44:30.677412 3815 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-wx9lc-82lcb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:44:31.550609 3815 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-wx9lc-82lcb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:44:31.550634 3815 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-wx9lc-82lcb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:44:38.157297 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 W0428 15:44:38.585172 3815 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0428 15:44:40.043917 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0428 15:44:41.610822 3815 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-w8nbv] creating logical port openshift-pipelines_pipelines-console-plugin-65f788bc74-w8nbv for pod on switch ip-10-0-0-99.ec2.internal I0428 15:44:41.610907 3815 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-65f788bc74-w8nbv I0428 15:44:41.626583 3815 pod.go:62] [openshift-pipelines/pipelines-console-plugin-65f788bc74-w8nbv] pod update took 15.695151ms I0428 15:44:41.626610 3815 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-w8nbv] addLogicalPort annotation time took 15.727669ms I0428 15:44:41.627900 3815 pods.go:271] [openshift-pipelines/pipelines-console-plugin-65f788bc74-w8nbv] addLogicalPort took 17.094794ms, libovsdb time 858.766µs I0428 15:44:42.581966 3815 cni.go:452] [openshift-pipelines/pipelines-console-plugin-65f788bc74-w8nbv 72e892d54bdfbedea249fa47e4e688246a5233246ac23e18c2b22d4712d8f38b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-w8nbv 72e892d54bdfbedea249fa47e4e688246a5233246ac23e18c2b22d4712d8f38b network default NAD default NAD key ] I0428 15:44:42.591767 3815 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-65f788bc74-w8nbv, hostIfaceName: 72e892d54bdfbed, network: default, NAD default, SandboxID: "72e892d54bdfbedea249fa47e4e688246a5233246ac23e18c2b22d4712d8f38b", PCI device ID: , UID: "7191bcc5-a2be-4e9f-8230-a7ebaf2cb774", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0428 15:44:42.644262 3815 cni.go:473] [openshift-pipelines/pipelines-console-plugin-65f788bc74-w8nbv 72e892d54bdfbedea249fa47e4e688246a5233246ac23e18c2b22d4712d8f38b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-w8nbv 72e892d54bdfbedea249fa47e4e688246a5233246ac23e18c2b22d4712d8f38b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:2d:2b:7d:0c:e1\",\"name\":\"72e892d54bdfbed\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1db27347-afa5-49ca-b969-dfe7e0322a29\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:44:42.950543 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=91 I0428 15:45:07.927270 3815 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:07.927829 3815 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:07.949299 3815 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:07.949315 3815 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:07.956430 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0428 15:45:07.956456 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0428 15:45:08.177592 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0428 15:45:15.959465 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=1009 I0428 15:45:16.981372 3815 obj_retry.go:498] Detected object tsf/patch-tekton-config-l8clm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:16.981970 3815 obj_retry.go:498] Detected object tsf/patch-tekton-config-l8clm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:18.003683 3815 obj_retry.go:492] Detected object tsf/patch-tekton-config-l8clm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:18.003707 3815 obj_retry.go:492] Detected object tsf/patch-tekton-config-l8clm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:22.830230 3815 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:45:24.019608 3815 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-45n9f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:24.020209 3815 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-45n9f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:25.052617 3815 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-45n9f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:25.052643 3815 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-45n9f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:25.095130 3815 obj_retry.go:492] Detected object tsf/patch-tekton-config-l8clm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:25.095156 3815 obj_retry.go:492] Detected object tsf/patch-tekton-config-l8clm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:25.100353 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-l8clm I0428 15:45:25.100380 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-l8clm I0428 15:45:29.052533 3815 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:29.053180 3815 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:29.074455 3815 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:29.074472 3815 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:29.081962 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0428 15:45:29.081985 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0428 15:45:29.163782 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0428 15:45:29.964459 3815 cni.go:452] [tsf-tas/rekor-createtree-job-6w4l5-548k2 824ea037ae92b8ac89ad13aee8b131ee5178d9dee902b137b9bd010c15653d54 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-6w4l5-548k2 824ea037ae92b8ac89ad13aee8b131ee5178d9dee902b137b9bd010c15653d54 network default NAD default NAD key ] I0428 15:45:30.042504 3815 cni.go:473] [tsf-tas/rekor-createtree-job-6w4l5-548k2 824ea037ae92b8ac89ad13aee8b131ee5178d9dee902b137b9bd010c15653d54 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-6w4l5-548k2 824ea037ae92b8ac89ad13aee8b131ee5178d9dee902b137b9bd010c15653d54 network default NAD default NAD key default], result "{}", err I0428 15:45:30.093383 3815 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-6w4l5-548k2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:30.093420 3815 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-6w4l5-548k2 I0428 15:45:30.094901 3815 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-6w4l5-548k2, ips: 10.129.0.38 I0428 15:45:30.094932 3815 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-6w4l5-548k2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:30.484268 3815 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-r2wnb] creating logical port tsf-tpa_create-db-r2wnb for pod on switch ip-10-0-0-99.ec2.internal I0428 15:45:30.484361 3815 kube.go:256] Updating pod tsf-tpa/create-db-r2wnb I0428 15:45:30.498222 3815 pod.go:62] [tsf-tpa/create-db-r2wnb] pod update took 13.884162ms I0428 15:45:30.498262 3815 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-r2wnb] addLogicalPort annotation time took 13.929028ms I0428 15:45:30.499617 3815 pods.go:271] [tsf-tpa/create-db-r2wnb] addLogicalPort took 15.394783ms, libovsdb time 992.507µs I0428 15:45:30.842511 3815 cni.go:452] [tsf-tpa/create-db-r2wnb 94cbb57523dbab23ebb998cf645a8b2dd63fb92a95bd99c2199a7f2cabf1d422 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-r2wnb 94cbb57523dbab23ebb998cf645a8b2dd63fb92a95bd99c2199a7f2cabf1d422 network default NAD default NAD key ] I0428 15:45:30.852294 3815 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-r2wnb, hostIfaceName: 94cbb57523dbab2, network: default, NAD default, SandboxID: "94cbb57523dbab23ebb998cf645a8b2dd63fb92a95bd99c2199a7f2cabf1d422", PCI device ID: , UID: "38bd0e1b-d5f0-4f0e-949a-69b6099f5023", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0428 15:45:30.905572 3815 cni.go:473] [tsf-tpa/create-db-r2wnb 94cbb57523dbab23ebb998cf645a8b2dd63fb92a95bd99c2199a7f2cabf1d422 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-r2wnb 94cbb57523dbab23ebb998cf645a8b2dd63fb92a95bd99c2199a7f2cabf1d422 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:15:36:ee:0e:2d\",\"name\":\"94cbb57523dbab2\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/42402051-242e-435e-a4fb-28ecffbf9192\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:45:30.980766 3815 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-6w4l5-548k2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:30.980786 3815 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-6w4l5-548k2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:34.997508 3815 cni.go:452] [tsf-tpa/create-db-r2wnb 94cbb57523dbab23ebb998cf645a8b2dd63fb92a95bd99c2199a7f2cabf1d422 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-r2wnb 94cbb57523dbab23ebb998cf645a8b2dd63fb92a95bd99c2199a7f2cabf1d422 network default NAD default NAD key ] I0428 15:45:35.076172 3815 cni.go:473] [tsf-tpa/create-db-r2wnb 94cbb57523dbab23ebb998cf645a8b2dd63fb92a95bd99c2199a7f2cabf1d422 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-r2wnb 94cbb57523dbab23ebb998cf645a8b2dd63fb92a95bd99c2199a7f2cabf1d422 network default NAD default NAD key default], result "{}", err I0428 15:45:35.135576 3815 obj_retry.go:498] Detected object tsf-tpa/create-db-r2wnb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:35.135607 3815 pods.go:176] Deleting pod: tsf-tpa/create-db-r2wnb I0428 15:45:35.136968 3815 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-r2wnb, ips: 10.129.0.40 I0428 15:45:35.136997 3815 obj_retry.go:498] Detected object tsf-tpa/create-db-r2wnb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:36.002256 3815 cni.go:452] [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d 7cfa1bfef90b953775637f8c4cac1fb1788f4f89fb5ff16b981d3f29825fb806 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d 7cfa1bfef90b953775637f8c4cac1fb1788f4f89fb5ff16b981d3f29825fb806 network default NAD default NAD key ] I0428 15:45:36.082216 3815 cni.go:473] [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d 7cfa1bfef90b953775637f8c4cac1fb1788f4f89fb5ff16b981d3f29825fb806 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-xpjqx-hkq2d 7cfa1bfef90b953775637f8c4cac1fb1788f4f89fb5ff16b981d3f29825fb806 network default NAD default NAD key default], result "{}", err I0428 15:45:36.137339 3815 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-xpjqx-hkq2d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:36.137375 3815 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-xpjqx-hkq2d I0428 15:45:36.138810 3815 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-xpjqx-hkq2d, ips: 10.129.0.37 I0428 15:45:36.138839 3815 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-xpjqx-hkq2d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:36.160602 3815 obj_retry.go:492] Detected object tsf-tpa/create-db-r2wnb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:36.160623 3815 obj_retry.go:492] Detected object tsf-tpa/create-db-r2wnb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:37.028160 3815 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-xpjqx-hkq2d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:37.028188 3815 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-xpjqx-hkq2d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:38.639862 3815 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-67ff9cb5fd-8bfhv] creating logical port tsf-tas_ctlog-67ff9cb5fd-8bfhv for pod on switch ip-10-0-0-99.ec2.internal I0428 15:45:38.639990 3815 kube.go:256] Updating pod tsf-tas/ctlog-67ff9cb5fd-8bfhv I0428 15:45:38.652836 3815 pod.go:62] [tsf-tas/ctlog-67ff9cb5fd-8bfhv] pod update took 12.872464ms I0428 15:45:38.652861 3815 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-67ff9cb5fd-8bfhv] addLogicalPort annotation time took 12.902456ms I0428 15:45:38.654442 3815 pods.go:271] [tsf-tas/ctlog-67ff9cb5fd-8bfhv] addLogicalPort took 14.599163ms, libovsdb time 999.841µs W0428 15:45:38.936690 3815 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0428 15:45:39.001214 3815 cni.go:452] [tsf-tas/ctlog-67ff9cb5fd-8bfhv be01888536875e63645d81d8e5a8a3fddf0e031f37ccc3aa79a5510659b3732e network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-67ff9cb5fd-8bfhv be01888536875e63645d81d8e5a8a3fddf0e031f37ccc3aa79a5510659b3732e network default NAD default NAD key ] I0428 15:45:39.011147 3815 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-67ff9cb5fd-8bfhv, hostIfaceName: be01888536875e6, network: default, NAD default, SandboxID: "be01888536875e63645d81d8e5a8a3fddf0e031f37ccc3aa79a5510659b3732e", PCI device ID: , UID: "71265fe2-d016-485e-a505-fa36bc50c1fb", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0428 15:45:39.066500 3815 cni.go:473] [tsf-tas/ctlog-67ff9cb5fd-8bfhv be01888536875e63645d81d8e5a8a3fddf0e031f37ccc3aa79a5510659b3732e network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-67ff9cb5fd-8bfhv be01888536875e63645d81d8e5a8a3fddf0e031f37ccc3aa79a5510659b3732e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:7f:4c:12:83:03\",\"name\":\"be01888536875e6\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f664d6c6-53f7-4e57-bd80-70fd9f304483\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:45:39.979837 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0428 15:45:41.030723 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0428 15:45:42.111328 3815 obj_retry.go:498] Detected object tsf-tpa/migrate-db-c9d9m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:42.111890 3815 obj_retry.go:498] Detected object tsf-tpa/migrate-db-c9d9m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:42.979300 3815 obj_retry.go:492] Detected object tsf-tpa/migrate-db-c9d9m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:42.979328 3815 obj_retry.go:492] Detected object tsf-tpa/migrate-db-c9d9m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:45.129486 3815 obj_retry.go:498] Detected object tsf-tpa/create-importers-w29pd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:45.130097 3815 obj_retry.go:498] Detected object tsf-tpa/create-importers-w29pd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0428 15:45:46.020619 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0428 15:45:46.106074 3815 base_network_controller_pods.go:487] [default/tsf-tpa/server-85ffdb4759-wqjdt] creating logical port tsf-tpa_server-85ffdb4759-wqjdt for pod on switch ip-10-0-0-99.ec2.internal I0428 15:45:46.106184 3815 kube.go:256] Updating pod tsf-tpa/server-85ffdb4759-wqjdt I0428 15:45:46.127656 3815 pod.go:62] [tsf-tpa/server-85ffdb4759-wqjdt] pod update took 21.498796ms I0428 15:45:46.127681 3815 base_network_controller_pods.go:951] [default/tsf-tpa/server-85ffdb4759-wqjdt] addLogicalPort annotation time took 21.531364ms I0428 15:45:46.129011 3815 pods.go:271] [tsf-tpa/server-85ffdb4759-wqjdt] addLogicalPort took 22.954331ms, libovsdb time 931.663µs I0428 15:45:46.155553 3815 obj_retry.go:492] Detected object tsf-tpa/create-importers-w29pd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:46.155576 3815 obj_retry.go:492] Detected object tsf-tpa/create-importers-w29pd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:46.162886 3815 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-lcb4j] creating logical port tsf-tpa_importer-86bcdf477b-lcb4j for pod on switch ip-10-0-0-99.ec2.internal I0428 15:45:46.163024 3815 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-lcb4j I0428 15:45:46.177434 3815 pod.go:62] [tsf-tpa/importer-86bcdf477b-lcb4j] pod update took 14.437538ms I0428 15:45:46.177458 3815 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-lcb4j] addLogicalPort annotation time took 14.466622ms I0428 15:45:46.178976 3815 pods.go:271] [tsf-tpa/importer-86bcdf477b-lcb4j] addLogicalPort took 16.110752ms, libovsdb time 1.05635ms W0428 15:45:52.044797 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0428 15:45:56.667430 3815 cni.go:452] [tsf-tpa/server-85ffdb4759-wqjdt 5254d013a8cba0f10752b01bfb8c7efd7884632a68f424ee77e240af77e2e94c network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-85ffdb4759-wqjdt 5254d013a8cba0f10752b01bfb8c7efd7884632a68f424ee77e240af77e2e94c network default NAD default NAD key ] I0428 15:45:56.677110 3815 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-85ffdb4759-wqjdt, hostIfaceName: 5254d013a8cba0f, network: default, NAD default, SandboxID: "5254d013a8cba0f10752b01bfb8c7efd7884632a68f424ee77e240af77e2e94c", PCI device ID: , UID: "d8d9c4bb-7f94-4483-8c5c-32940befe4e6", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0428 15:45:56.731003 3815 cni.go:452] [tsf-tpa/importer-86bcdf477b-lcb4j 5f454db0bd958d294559f6c41fa22e7650d61103aec8928b38c1220b664f290e network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-lcb4j 5f454db0bd958d294559f6c41fa22e7650d61103aec8928b38c1220b664f290e network default NAD default NAD key ] I0428 15:45:56.735501 3815 cni.go:473] [tsf-tpa/server-85ffdb4759-wqjdt 5254d013a8cba0f10752b01bfb8c7efd7884632a68f424ee77e240af77e2e94c network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-85ffdb4759-wqjdt 5254d013a8cba0f10752b01bfb8c7efd7884632a68f424ee77e240af77e2e94c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:73:1f:56:bb:5c\",\"name\":\"5254d013a8cba0f\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d94760a5-c6c4-42da-80bb-0671c26782fe\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:45:56.740849 3815 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-lcb4j, hostIfaceName: 5f454db0bd958d2, network: default, NAD default, SandboxID: "5f454db0bd958d294559f6c41fa22e7650d61103aec8928b38c1220b664f290e", PCI device ID: , UID: "7082e690-549e-4dd4-a36f-c4020aee4175", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0428 15:45:56.799300 3815 cni.go:473] [tsf-tpa/importer-86bcdf477b-lcb4j 5f454db0bd958d294559f6c41fa22e7650d61103aec8928b38c1220b664f290e network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-lcb4j 5f454db0bd958d294559f6c41fa22e7650d61103aec8928b38c1220b664f290e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:c5:4a:de:4d:7d\",\"name\":\"5f454db0bd958d2\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2d92617d-2a7b-442c-92a3-a09e2eed303b\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:45:57.148107 3815 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0428 15:45:58.148398 3815 util.go:959] Failed to build node endpoints for node ip-10-0-0-99.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0428 15:46:15.322886 3815 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:15.323455 3815 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:15.364657 3815 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:15.364686 3815 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:15.375601 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0428 15:46:15.375627 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0428 15:46:16.162771 3815 namespace.go:98] [default-tenant] adding namespace I0428 15:46:16.163666 3815 namespace.go:102] [default-tenant] adding namespace took 870.792µs I0428 15:46:16.167103 3815 namespace.go:98] [image-controller] adding namespace I0428 15:46:16.167183 3815 namespace.go:98] [build-service] adding namespace I0428 15:46:16.167950 3815 namespace.go:102] [image-controller] adding namespace took 829.882µs I0428 15:46:16.168566 3815 namespace.go:102] [build-service] adding namespace took 1.369425ms I0428 15:46:16.174059 3815 namespace.go:98] [integration-service] adding namespace I0428 15:46:16.174830 3815 namespace.go:102] [integration-service] adding namespace took 752.537µs I0428 15:46:16.253576 3815 namespace.go:142] [default-tenant] updating namespace I0428 15:46:16.301175 3815 namespace.go:142] [default-tenant] updating namespace I0428 15:46:16.313024 3815 namespace.go:142] [image-controller] updating namespace I0428 15:46:16.349478 3815 namespace.go:142] [build-service] updating namespace I0428 15:46:16.349592 3815 namespace.go:142] [image-controller] updating namespace I0428 15:46:16.379300 3815 namespace.go:142] [build-service] updating namespace I0428 15:46:16.390011 3815 namespace.go:142] [integration-service] updating namespace I0428 15:46:16.431277 3815 namespace.go:142] [integration-service] updating namespace I0428 15:46:16.456882 3815 namespace.go:142] [build-service] updating namespace I0428 15:46:16.469544 3815 namespace.go:142] [integration-service] updating namespace I0428 15:46:16.498311 3815 namespace.go:98] [release-service] adding namespace I0428 15:46:16.499208 3815 namespace.go:102] [release-service] adding namespace took 871.385µs I0428 15:46:16.546183 3815 namespace.go:142] [release-service] updating namespace I0428 15:46:16.546342 3815 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:16.568871 3815 namespace.go:142] [release-service] updating namespace I0428 15:46:16.755021 3815 namespace.go:98] [konflux-info] adding namespace I0428 15:46:16.755765 3815 namespace.go:102] [konflux-info] adding namespace took 723.986µs I0428 15:46:16.809232 3815 namespace.go:142] [konflux-info] updating namespace I0428 15:46:16.835493 3815 namespace.go:98] [namespace-lister] adding namespace I0428 15:46:16.836355 3815 namespace.go:102] [namespace-lister] adding namespace took 843.731µs I0428 15:46:16.860036 3815 namespace.go:142] [konflux-info] updating namespace I0428 15:46:16.860132 3815 namespace.go:98] [enterprise-contract-service] adding namespace I0428 15:46:16.860767 3815 namespace.go:102] [enterprise-contract-service] adding namespace took 607.052µs I0428 15:46:16.931078 3815 namespace.go:142] [namespace-lister] updating namespace I0428 15:46:16.956784 3815 namespace.go:142] [image-controller] updating namespace I0428 15:46:16.983129 3815 namespace.go:142] [namespace-lister] updating namespace I0428 15:46:16.993602 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:46:17.075762 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:46:17.141140 3815 namespace.go:142] [default-tenant] updating namespace I0428 15:46:17.404361 3815 namespace.go:98] [segment-bridge] adding namespace I0428 15:46:17.405167 3815 namespace.go:102] [segment-bridge] adding namespace took 785.191µs I0428 15:46:17.463811 3815 namespace.go:98] [konflux-cli] adding namespace I0428 15:46:17.464605 3815 namespace.go:102] [konflux-cli] adding namespace took 774.384µs I0428 15:46:17.534216 3815 namespace.go:142] [segment-bridge] updating namespace I0428 15:46:17.585378 3815 namespace.go:142] [konflux-cli] updating namespace I0428 15:46:17.612713 3815 namespace.go:142] [segment-bridge] updating namespace I0428 15:46:17.722652 3815 namespace.go:142] [konflux-cli] updating namespace I0428 15:46:18.079834 3815 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0428 15:46:18.080482 3815 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0428 15:46:18.080512 3815 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0428 15:46:18.083268 3815 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0428 15:46:18.281325 3815 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0428 15:46:18.281358 3815 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0428 15:46:18.282200 3815 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0428 15:46:20.328332 3815 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-l8fb6-mfxdg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:20.328870 3815 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-l8fb6-mfxdg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:20.581259 3815 namespace.go:142] [build-service] updating namespace W0428 15:46:21.497655 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0428 15:46:21.647160 3815 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-l8fb6-mfxdg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:21.647188 3815 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-l8fb6-mfxdg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:21.804259 3815 namespace.go:142] [default-tenant] updating namespace I0428 15:46:22.419585 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:46:22.776302 3815 namespace.go:142] [image-controller] updating namespace I0428 15:46:23.080286 3815 namespace.go:142] [integration-service] updating namespace I0428 15:46:23.510632 3815 namespace.go:142] [konflux-info] updating namespace I0428 15:46:24.134220 3815 namespace.go:142] [namespace-lister] updating namespace I0428 15:46:24.827916 3815 namespace.go:142] [release-service] updating namespace I0428 15:46:25.301053 3815 namespace.go:142] [segment-bridge] updating namespace I0428 15:46:25.571214 3815 namespace.go:142] [build-service] updating namespace I0428 15:46:25.674613 3815 namespace.go:142] [default-tenant] updating namespace I0428 15:46:25.694133 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:46:25.722121 3815 namespace.go:142] [image-controller] updating namespace I0428 15:46:25.747835 3815 namespace.go:142] [integration-service] updating namespace I0428 15:46:25.784713 3815 namespace.go:142] [konflux-info] updating namespace I0428 15:46:25.847132 3815 namespace.go:142] [namespace-lister] updating namespace I0428 15:46:25.891090 3815 namespace.go:142] [release-service] updating namespace I0428 15:46:25.951028 3815 namespace.go:142] [segment-bridge] updating namespace I0428 15:46:26.074805 3815 namespace.go:142] [build-service] updating namespace I0428 15:46:26.173042 3815 namespace.go:142] [default-tenant] updating namespace W0428 15:46:26.288385 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0428 15:46:26.328465 3815 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:46:26.418124 3815 namespace.go:142] [image-controller] updating namespace I0428 15:46:26.516214 3815 namespace.go:142] [integration-service] updating namespace I0428 15:46:26.641286 3815 namespace.go:142] [konflux-info] updating namespace I0428 15:46:26.764132 3815 namespace.go:142] [namespace-lister] updating namespace I0428 15:46:26.893460 3815 namespace.go:142] [release-service] updating namespace I0428 15:46:26.957670 3815 namespace.go:142] [segment-bridge] updating namespace W0428 15:46:28.268341 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 15:46:28.269370 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service I0428 15:46:28.285176 3815 namespace.go:142] [konflux-cli] updating namespace I0428 15:46:28.414266 3815 namespace.go:142] [konflux-cli] updating namespace I0428 15:46:28.510875 3815 namespace.go:142] [konflux-cli] updating namespace W0428 15:46:29.284112 3815 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0428 15:46:29.311452 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 15:46:29.319013 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 15:46:29.332314 3815 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0428 15:46:32.292949 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 15:46:32.292972 3815 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 15:46:32.354229 3815 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 15:46:32.354264 3815 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 15:46:34.309687 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0428 15:46:34.309800 3815 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0428 15:46:35.304714 3815 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0428 15:46:40.667711 3815 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-l5jlx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:40.668282 3815 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-l5jlx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:41.342470 3815 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-l5jlx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:41.342492 3815 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-l5jlx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:41.354085 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-l5jlx I0428 15:46:41.354114 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-l5jlx I0428 15:47:42.354981 3815 obj_retry.go:498] Detected object konflux-ui/proxy-6b7d95fccd-fkzn4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:42.355575 3815 obj_retry.go:498] Detected object konflux-ui/proxy-6b7d95fccd-fkzn4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:42.734488 3815 obj_retry.go:492] Detected object konflux-ui/proxy-6b7d95fccd-fkzn4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:42.734516 3815 obj_retry.go:492] Detected object konflux-ui/proxy-6b7d95fccd-fkzn4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:42.743191 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-6b7d95fccd-fkzn4 I0428 15:47:42.743218 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-6b7d95fccd-fkzn4 I0428 15:47:42.967877 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=116 I0428 15:47:46.850077 3815 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:46.850682 3815 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:46.870605 3815 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:46.870627 3815 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:46.874552 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0428 15:47:46.874579 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0428 15:48:19.962200 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0428 15:48:20.813079 3815 namespace.go:98] [managed-tenant-mgvo0] adding namespace I0428 15:48:20.814023 3815 namespace.go:102] [managed-tenant-mgvo0] adding namespace took 916.8µs I0428 15:48:20.837275 3815 namespace.go:142] [managed-tenant-mgvo0] updating namespace I0428 15:48:20.860471 3815 namespace.go:142] [managed-tenant-mgvo0] updating namespace I0428 15:48:21.173562 3815 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0428 15:48:22.088490 3815 namespace.go:142] [managed-tenant-mgvo0] updating namespace I0428 15:48:22.219166 3815 namespace.go:142] [managed-tenant-mgvo0] updating namespace I0428 15:48:22.327560 3815 namespace.go:142] [managed-tenant-mgvo0] updating namespace I0428 15:48:51.046434 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0428 15:50:04.573484 3815 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-5jzn7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:04.574070 3815 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-5jzn7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:05.443876 3815 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-5jzn7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:05.443904 3815 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-5jzn7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:16.153652 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0428 15:50:18.179497 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0428 15:50:24.047109 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=5 I0428 15:50:33.953427 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=33 I0428 15:50:55.961343 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=54 I0428 15:50:57.058957 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0428 15:51:16.154749 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0428 15:51:30.961815 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=233 I0428 15:51:56.159987 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0428 15:52:17.953693 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=98 I0428 15:53:06.181232 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0428 15:54:03.167694 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0428 15:55:36.053775 3815 obj_retry.go:492] Detected object tsf-tpa/create-db-r2wnb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:55:36.053835 3815 obj_retry.go:492] Detected object tsf-tpa/create-db-r2wnb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:55:36.059632 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-r2wnb I0428 15:55:36.059660 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-r2wnb I0428 15:55:42.037174 3815 obj_retry.go:492] Detected object tsf-tpa/migrate-db-c9d9m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:55:42.037232 3815 obj_retry.go:492] Detected object tsf-tpa/migrate-db-c9d9m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:55:42.045499 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-c9d9m I0428 15:55:42.045525 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-c9d9m I0428 15:55:46.034792 3815 obj_retry.go:492] Detected object tsf-tpa/create-importers-w29pd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:55:46.034859 3815 obj_retry.go:492] Detected object tsf-tpa/create-importers-w29pd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:55:46.040757 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-w29pd I0428 15:55:46.040782 3815 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-w29pd I0428 15:55:52.156806 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0428 15:55:57.049356 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0428 15:56:03.177260 3815 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0428 15:56:09.956048 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=32 I0428 15:56:24.965476 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0428 15:56:32.970556 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=16 I0428 15:56:53.050624 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0428 15:57:10.182573 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0428 15:57:21.964755 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=12 I0428 15:57:54.162723 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0428 15:59:00.956368 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=7 I0428 15:59:38.157816 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0428 15:59:52.061781 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0428 16:00:03.556780 3815 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-rcstc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 16:00:03.557393 3815 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-rcstc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 16:00:04.584907 3815 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-rcstc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:04.584929 3815 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-rcstc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:28.964343 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0428 16:01:25.052200 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0428 16:01:28.184448 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0428 16:01:40.164563 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0428 16:02:13.185758 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0428 16:02:23.967136 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=27 I0428 16:02:42.170131 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0428 16:02:55.165785 3815 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=5 I0428 16:02:57.180175 3815 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0428 16:03:02.973303 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7 I0428 16:03:36.053373 3815 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0428 16:04:00.959377 3815 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=42