+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-186.ec2.internal ++ [[ -n ip-10-0-2-186.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-186.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0425 23:02:11.437027522 - disable conntrack on geneve port' I0425 23:02:11.437027522 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0425 23:02:11.463046072 - starting ovnkube-node + echo 'I0425 23:02:11.463046072 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-186.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-186.ec2.internal --init-node ip-10-0-2-186.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-186.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0425 23:02:11.508292 3778 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0425 23:02:11.508349 3778 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-bb77b33dba.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0425 23:02:11.509859 3778 kube.go:431] Waiting for certificate I0425 23:02:11.509870 3778 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0425 23:02:11.509898 3778 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0425 23:02:11.510622 3778 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0425 23:02:11.541179 3778 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0425 23:02:11.541194 3778 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0425 23:02:11.543622 3778 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0425 23:02:11.543709 3778 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-b8v5n" I0425 23:02:11.548864 3778 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-b8v5n" I0425 23:02:11.548918 3778 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0425 23:02:12.510564 3778 kube.go:438] Certificate found I0425 23:02:12.511193 3778 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0425 23:02:12.512236 3778 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0425 23:02:12.562288 3778 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-26 22:57:11 +0000 UTC" deadline="2026-04-26 18:39:59.701670286 +0000 UTC" I0425 23:02:12.562327 3778 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h37m47.139356772s" I0425 23:02:12.573791 3778 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0425 23:02:12.593751 3778 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0425 23:02:12.593815 3778 server.go:101] MetricServer registers OVS metrics I0425 23:02:12.593815 3778 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0425 23:02:12.593933 3778 factory.go:561] Starting watch factory I0425 23:02:12.594030 3778 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594046 3778 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594048 3778 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594052 3778 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594123 3778 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594204 3778 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594228 3778 server.go:105] MetricServer registers OVN DB metrics I0425 23:02:12.594129 3778 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594306 3778 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594135 3778 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594385 3778 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594418 3778 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.594059 3778 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.597947 3778 controller_manager.go:370] Starting the ovnkube controller I0425 23:02:12.597961 3778 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-186.ec2.internal I0425 23:02:12.598023 3778 controller_manager.go:395] NBDB zone sync took: 49.612µs I0425 23:02:12.598038 3778 factory.go:561] Starting watch factory I0425 23:02:12.599146 3778 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0425 23:02:12.599166 3778 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0425 23:02:12.613497 3778 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0425 23:02:12.613516 3778 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0425 23:02:12.617374 3778 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0425 23:02:12.617407 3778 server.go:109] MetricServer registers OVN Controller metrics I0425 23:02:12.617443 3778 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.620774 3778 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.621012 3778 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.621285 3778 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.622444 3778 server.go:113] MetricServer registers OVN Northd metrics I0425 23:02:12.624727 3778 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.626863 3778 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0425 23:02:12.628407 3778 metrics.go:519] OVN Metrics Server starts to run ... I0425 23:02:12.699593 3778 factory.go:1977] *v1.Node informer cache synced successfully I0425 23:02:12.699608 3778 factory.go:1977] *v1.Service informer cache synced successfully I0425 23:02:12.699612 3778 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0425 23:02:12.699615 3778 factory.go:1977] *v1.Pod informer cache synced successfully I0425 23:02:12.699618 3778 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0425 23:02:12.699621 3778 factory.go:1977] *v1.Namespace informer cache synced successfully I0425 23:02:12.699682 3778 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 23:02:12.699695 3778 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 23:02:12.699726 3778 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 23:02:12.699747 3778 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 23:02:12.703211 3778 factory.go:1977] *v1.Service informer cache synced successfully I0425 23:02:12.703220 3778 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0425 23:02:12.703223 3778 factory.go:1977] *v1.Pod informer cache synced successfully I0425 23:02:12.703226 3778 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0425 23:02:12.703229 3778 factory.go:1977] *v1.Namespace informer cache synced successfully I0425 23:02:12.703232 3778 factory.go:1977] *v1.Node informer cache synced successfully I0425 23:02:12.703584 3778 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 23:02:12.703591 3778 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 23:02:12.705416 3778 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0425 23:02:12.705427 3778 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0425 23:02:12.705474 3778 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.705485 3778 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.708251 3778 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0425 23:02:12.708260 3778 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0425 23:02:12.708417 3778 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.711246 3778 factory.go:1977] *v1.EgressIP informer cache synced successfully I0425 23:02:12.711302 3778 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.711313 3778 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.713452 3778 factory.go:1977] *v1.EgressIP informer cache synced successfully I0425 23:02:12.714663 3778 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.716703 3778 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0425 23:02:12.716756 3778 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.716768 3778 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.718507 3778 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0425 23:02:12.720060 3778 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.722077 3778 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0425 23:02:12.722138 3778 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.722148 3778 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.723809 3778 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0425 23:02:12.725006 3778 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.727694 3778 factory.go:1977] *v1.EgressService informer cache synced successfully I0425 23:02:12.727800 3778 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.727819 3778 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.729895 3778 factory.go:1977] *v1.EgressService informer cache synced successfully I0425 23:02:12.730926 3778 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.732885 3778 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0425 23:02:12.733210 3778 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0425 23:02:12.733231 3778 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0425 23:02:12.735768 3778 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0425 23:02:12.736216 3778 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0425 23:02:12.738206 3778 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0425 23:02:12.738261 3778 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.738275 3778 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.738323 3778 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.738339 3778 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.741073 3778 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0425 23:02:12.741404 3778 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.741463 3778 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 23:02:12.743490 3778 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0425 23:02:12.743500 3778 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0425 23:02:12.743507 3778 factory.go:680] Watch Factory start up complete, took: 149.579142ms I0425 23:02:12.743569 3778 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0425 23:02:12.746984 3778 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0425 23:02:12.746994 3778 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0425 23:02:12.747000 3778 factory.go:680] Watch Factory start up complete, took: 148.962163ms I0425 23:02:12.747007 3778 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-186.ec2.internal" zone I0425 23:02:12.789043 3778 default_node_network_controller.go:710] Initializing the default node network controller I0425 23:02:12.880353 3778 udn_isolation.go:101] Starting UDN host isolation manager I0425 23:02:12.887085 3778 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0425 23:02:12.949347 3778 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0425 23:02:12.949417 3778 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0425 23:02:12.949434 3778 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0425 23:02:12.982449 3778 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0425 23:02:12.982506 3778 default_node_network_controller.go:809] Node ip-10-0-2-186.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0425 23:02:12.987641 3778 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.186"] k8s.ovn.org/zone-name:ip-10-0-2-186.ec2.internal] on node ip-10-0-2-186.ec2.internal I0425 23:02:13.007949 3778 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0425 23:02:13.013900 3778 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0425 23:02:13.014016 3778 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0425 23:02:13.014136 3778 gateway_init.go:263] Preparing Gateway I0425 23:02:13.014147 3778 gateway_shared_intf.go:1687] Creating new gateway I0425 23:02:13.109038 3778 iptables.go:108] Creating table: filter chain: FORWARD I0425 23:02:13.116073 3778 iptables.go:108] Creating table: filter chain: OUTPUT I0425 23:02:13.122915 3778 gateway_shared_intf.go:1816] Gateway Creation Complete I0425 23:02:13.122932 3778 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-186.ec2.internal","mac-address":"0e:97:be:5d:f0:43","ip-addresses":["10.0.2.186/24"],"ip-address":"10.0.2.186/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:2ed05171-23d6-4ed8-98cd-2bb1fbabb65b k8s.ovn.org/node-encap-ips:["10.0.2.186"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.186/24"} k8s.ovn.org/zone-name:ip-10-0-2-186.ec2.internal] on node ip-10-0-2-186.ec2.internal I0425 23:02:13.145043 3778 default_node_network_controller.go:519] Node connection status = connected I0425 23:02:13.247633 3778 controller_manager.go:429] Waiting for node in zone sync took: 500.613746ms I0425 23:02:13.257611 3778 controller_manager.go:611] Switching to transit router for layer2 networks I0425 23:02:13.257625 3778 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-186.ec2.internal I0425 23:02:13.279687 3778 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0425 23:02:13.287435 3778 controller_manager.go:290] SCTP support detected in OVN I0425 23:02:13.288058 3778 services_controller.go:69] Creating services controller for network=default I0425 23:02:13.288124 3778 udn_enabled_svc.go:80] Waiting for service informer to sync I0425 23:02:13.288209 3778 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0425 23:02:13.288239 3778 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0425 23:02:13.288251 3778 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0425 23:02:13.288259 3778 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0425 23:02:13.288421 3778 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0425 23:02:13.288469 3778 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0425 23:02:13.288518 3778 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 60.47µs I0425 23:02:13.288542 3778 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 65.458µs I0425 23:02:13.288548 3778 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 79.444µs I0425 23:02:13.288554 3778 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0425 23:02:13.288561 3778 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0425 23:02:13.288573 3778 default_network_controller.go:325] Starting the default network controller I0425 23:02:13.288593 3778 udn_enabled_svc.go:95] Performing full resync I0425 23:02:13.288679 3778 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 12.708µs I0425 23:02:13.288896 3778 udn_enabled_svc.go:99] Waiting for handler to sync I0425 23:02:13.288910 3778 udn_enabled_svc.go:104] Starting worker I0425 23:02:13.298783 3778 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0425 23:02:13.298839 3778 repair.go:33] Syncing exgw routes took 43.93µs I0425 23:02:13.298861 3778 default_network_controller.go:414] Starting all the Watchers... I0425 23:02:13.299005 3778 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0425 23:02:13.299042 3778 namespace.go:98] [openshift-ingress] adding namespace I0425 23:02:13.299046 3778 namespace.go:98] [openshift-must-gather-operator] adding namespace I0425 23:02:13.299043 3778 namespace.go:98] [openshift-ingress-canary] adding namespace I0425 23:02:13.299059 3778 namespace.go:98] [openshift-config-operator] adding namespace I0425 23:02:13.299063 3778 namespace.go:98] [openshift-backplane-mobb] adding namespace I0425 23:02:13.299069 3778 namespace.go:98] [open-cluster-management-2ptfd0doqkjckkqnviaqdhhsij5e416h] adding namespace I0425 23:02:13.299075 3778 namespace.go:98] [openshift-operators-redhat] adding namespace I0425 23:02:13.299070 3778 namespace.go:98] [openshift-network-diagnostics] adding namespace I0425 23:02:13.299087 3778 namespace.go:98] [openshift-console-user-settings] adding namespace I0425 23:02:13.299115 3778 namespace.go:98] [default] adding namespace I0425 23:02:13.299114 3778 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0425 23:02:13.299500 3778 namespace.go:98] [openshift-customer-monitoring] adding namespace I0425 23:02:13.299510 3778 namespace.go:98] [openshift-dns] adding namespace I0425 23:02:13.299532 3778 namespace.go:98] [openshift-machine-api] adding namespace I0425 23:02:13.299671 3778 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 653.441µs I0425 23:02:13.299686 3778 namespace.go:98] [openshift-package-operator] adding namespace I0425 23:02:13.300223 3778 namespace.go:102] [openshift-package-operator] adding namespace took 529.989µs I0425 23:02:13.300242 3778 namespace.go:98] [openshift-backplane-csm] adding namespace I0425 23:02:13.300767 3778 namespace.go:102] [openshift-backplane-csm] adding namespace took 514.67µs I0425 23:02:13.300786 3778 namespace.go:98] [openshift-monitoring] adding namespace I0425 23:02:13.301365 3778 namespace.go:102] [openshift-ingress] adding namespace took 2.312164ms I0425 23:02:13.301383 3778 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0425 23:02:13.301841 3778 namespace.go:102] [openshift-must-gather-operator] adding namespace took 2.785361ms I0425 23:02:13.301866 3778 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0425 23:02:13.302436 3778 namespace.go:102] [openshift-ingress-canary] adding namespace took 3.377123ms I0425 23:02:13.302451 3778 namespace.go:98] [openshift-controller-manager] adding namespace I0425 23:02:13.302952 3778 namespace.go:102] [openshift-config-operator] adding namespace took 3.885805ms I0425 23:02:13.302970 3778 namespace.go:98] [openshift-etcd] adding namespace I0425 23:02:13.303537 3778 namespace.go:102] [openshift-backplane-mobb] adding namespace took 4.46496ms I0425 23:02:13.303553 3778 namespace.go:98] [openshift-machine-config-operator] adding namespace I0425 23:02:13.304042 3778 namespace.go:102] [openshift-operators-redhat] adding namespace took 4.963076ms I0425 23:02:13.304052 3778 namespace.go:98] [openshift-authentication-operator] adding namespace W0425 23:02:13.304526 3778 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-d6vnf: pod openshift-network-diagnostics/network-check-target-d6vnf: no pod IPs found W0425 23:02:13.304544 3778 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-7qxbv: pod openshift-network-diagnostics/network-check-target-7qxbv: no pod IPs found I0425 23:02:13.304561 3778 namespace.go:102] [open-cluster-management-2ptfd0doqkjckkqnviaqdhhsij5e416h] adding namespace took 5.480818ms I0425 23:02:13.304575 3778 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0425 23:02:13.305139 3778 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.056476ms I0425 23:02:13.305153 3778 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0425 23:02:13.305722 3778 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.629077ms I0425 23:02:13.305736 3778 namespace.go:98] [openshift] adding namespace I0425 23:02:13.306237 3778 namespace.go:102] [default] adding namespace took 7.115033ms I0425 23:02:13.306250 3778 namespace.go:98] [openshift-multus] adding namespace I0425 23:02:13.306777 3778 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.649988ms I0425 23:02:13.306797 3778 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0425 23:02:13.307236 3778 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.724429ms I0425 23:02:13.307251 3778 namespace.go:98] [openshift-backplane-cse] adding namespace I0425 23:02:13.307789 3778 namespace.go:102] [openshift-dns] adding namespace took 8.272131ms I0425 23:02:13.307804 3778 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0425 23:02:13.308363 3778 namespace.go:102] [openshift-machine-api] adding namespace took 8.820799ms I0425 23:02:13.308376 3778 namespace.go:98] [openshift-kube-scheduler] adding namespace I0425 23:02:13.308826 3778 namespace.go:102] [openshift-monitoring] adding namespace took 8.035772ms I0425 23:02:13.308835 3778 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0425 23:02:13.309295 3778 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.904683ms I0425 23:02:13.309315 3778 namespace.go:98] [openshift-config-managed] adding namespace I0425 23:02:13.309710 3778 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.836688ms I0425 23:02:13.309729 3778 namespace.go:98] [openshift-host-network] adding namespace I0425 23:02:13.310156 3778 namespace.go:102] [openshift-controller-manager] adding namespace took 7.697046ms I0425 23:02:13.310173 3778 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0425 23:02:13.310689 3778 namespace.go:102] [openshift-etcd] adding namespace took 7.713922ms I0425 23:02:13.310700 3778 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0425 23:02:13.311112 3778 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.55216ms I0425 23:02:13.311128 3778 namespace.go:98] [openshift-authentication] adding namespace I0425 23:02:13.311527 3778 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.468858ms I0425 23:02:13.311541 3778 namespace.go:98] [kube-node-lease] adding namespace I0425 23:02:13.311943 3778 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.36177ms I0425 23:02:13.311963 3778 namespace.go:98] [openshift-backplane] adding namespace I0425 23:02:13.312395 3778 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.236523ms I0425 23:02:13.312411 3778 namespace.go:98] [openshift-console-operator] adding namespace W0425 23:02:13.312763 3778 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-6tvfc: pod openshift-multus/network-metrics-daemon-6tvfc: no pod IPs found W0425 23:02:13.312775 3778 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-k5q6r: pod openshift-multus/network-metrics-daemon-k5q6r: no pod IPs found I0425 23:02:13.312811 3778 namespace.go:102] [openshift] adding namespace took 7.069741ms I0425 23:02:13.312827 3778 namespace.go:98] [openshift-console] adding namespace I0425 23:02:13.313321 3778 namespace.go:102] [openshift-multus] adding namespace took 7.06566ms I0425 23:02:13.313337 3778 namespace.go:98] [openshift-backplane-tam] adding namespace I0425 23:02:13.313768 3778 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.965638ms I0425 23:02:13.313784 3778 namespace.go:98] [openshift-network-operator] adding namespace I0425 23:02:13.314217 3778 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.959888ms I0425 23:02:13.314230 3778 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0425 23:02:13.314654 3778 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.842334ms I0425 23:02:13.314675 3778 namespace.go:98] [kube-system] adding namespace I0425 23:02:13.315186 3778 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.802903ms I0425 23:02:13.315205 3778 namespace.go:98] [kube-public] adding namespace I0425 23:02:13.315626 3778 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.785398ms I0425 23:02:13.315642 3778 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0425 23:02:13.316107 3778 namespace.go:102] [openshift-config-managed] adding namespace took 6.78736ms I0425 23:02:13.316118 3778 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0425 23:02:13.316747 3778 namespace.go:102] [openshift-host-network] adding namespace took 7.011992ms I0425 23:02:13.316760 3778 namespace.go:98] [openshift-cluster-version] adding namespace I0425 23:02:13.317164 3778 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.986356ms I0425 23:02:13.317176 3778 namespace.go:98] [openshift-node] adding namespace I0425 23:02:13.317656 3778 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.950407ms I0425 23:02:13.317674 3778 namespace.go:98] [openshift-service-ca-operator] adding namespace I0425 23:02:13.318129 3778 namespace.go:102] [openshift-authentication] adding namespace took 6.993715ms I0425 23:02:13.318146 3778 namespace.go:98] [openshift-network-node-identity] adding namespace I0425 23:02:13.318714 3778 namespace.go:102] [kube-node-lease] adding namespace took 7.166533ms I0425 23:02:13.318730 3778 namespace.go:98] [dedicated-admin] adding namespace I0425 23:02:13.319182 3778 namespace.go:102] [openshift-backplane] adding namespace took 7.212909ms I0425 23:02:13.319198 3778 namespace.go:98] [openshift-ingress-operator] adding namespace I0425 23:02:13.319656 3778 namespace.go:102] [openshift-console-operator] adding namespace took 7.238337ms I0425 23:02:13.319670 3778 namespace.go:98] [openshift-logging] adding namespace I0425 23:02:13.320109 3778 namespace.go:102] [openshift-console] adding namespace took 7.258707ms I0425 23:02:13.320128 3778 namespace.go:98] [openshift-kube-apiserver] adding namespace I0425 23:02:13.320918 3778 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.5742ms I0425 23:02:13.320935 3778 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0425 23:02:13.321449 3778 namespace.go:102] [openshift-network-operator] adding namespace took 7.659559ms I0425 23:02:13.321465 3778 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0425 23:02:13.321909 3778 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.673398ms I0425 23:02:13.321924 3778 namespace.go:98] [openshift-marketplace] adding namespace I0425 23:02:13.322365 3778 namespace.go:102] [kube-system] adding namespace took 7.681114ms I0425 23:02:13.322384 3778 namespace.go:98] [openshift-backplane-srep] adding namespace I0425 23:02:13.322835 3778 namespace.go:102] [kube-public] adding namespace took 7.624246ms I0425 23:02:13.322850 3778 namespace.go:98] [openshift-operators] adding namespace I0425 23:02:13.323262 3778 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.614567ms I0425 23:02:13.323678 3778 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.554786ms I0425 23:02:13.323696 3778 namespace.go:98] [openshift-dns-operator] adding namespace I0425 23:02:13.324089 3778 namespace.go:102] [openshift-cluster-version] adding namespace took 7.324467ms I0425 23:02:13.324121 3778 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0425 23:02:13.324464 3778 namespace.go:102] [openshift-node] adding namespace took 7.282919ms I0425 23:02:13.324482 3778 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0425 23:02:13.325070 3778 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.390188ms I0425 23:02:13.325084 3778 namespace.go:98] [openshift-backplane-cee] adding namespace I0425 23:02:13.325500 3778 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.348777ms I0425 23:02:13.325510 3778 namespace.go:98] [openshift-apiserver-operator] adding namespace I0425 23:02:13.326042 3778 namespace.go:102] [dedicated-admin] adding namespace took 7.305373ms I0425 23:02:13.326062 3778 namespace.go:98] [openshift-insights] adding namespace I0425 23:02:13.326493 3778 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.287477ms I0425 23:02:13.326510 3778 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0425 23:02:13.326913 3778 namespace.go:102] [openshift-logging] adding namespace took 7.236776ms I0425 23:02:13.326930 3778 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0425 23:02:13.327408 3778 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.272334ms I0425 23:02:13.327420 3778 namespace.go:98] [openshift-route-controller-manager] adding namespace I0425 23:02:13.327796 3778 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.855727ms I0425 23:02:13.327809 3778 namespace.go:98] [openshift-config] adding namespace I0425 23:02:13.328189 3778 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.719777ms I0425 23:02:13.328200 3778 namespace.go:98] [openshift-apiserver] adding namespace I0425 23:02:13.328573 3778 namespace.go:102] [openshift-marketplace] adding namespace took 6.644687ms I0425 23:02:13.328585 3778 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0425 23:02:13.328933 3778 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.544569ms I0425 23:02:13.328944 3778 namespace.go:98] [openshift-infra] adding namespace I0425 23:02:13.329333 3778 namespace.go:102] [openshift-operators] adding namespace took 6.478532ms I0425 23:02:13.329694 3778 namespace.go:102] [openshift-dns-operator] adding namespace took 5.993443ms I0425 23:02:13.329706 3778 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0425 23:02:13.330059 3778 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 5.933288ms I0425 23:02:13.330418 3778 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.931818ms I0425 23:02:13.330431 3778 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0425 23:02:13.330846 3778 namespace.go:102] [openshift-backplane-cee] adding namespace took 5.756602ms I0425 23:02:13.331273 3778 namespace.go:102] [openshift-apiserver-operator] adding namespace took 5.757234ms I0425 23:02:13.331290 3778 namespace.go:98] [openshift-network-console] adding namespace I0425 23:02:13.331646 3778 namespace.go:102] [openshift-insights] adding namespace took 5.577291ms I0425 23:02:13.332035 3778 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 5.51885ms I0425 23:02:13.332434 3778 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 5.498914ms I0425 23:02:13.332820 3778 namespace.go:102] [openshift-route-controller-manager] adding namespace took 5.393258ms I0425 23:02:13.333207 3778 namespace.go:102] [openshift-config] adding namespace took 5.393077ms I0425 23:02:13.333663 3778 namespace.go:102] [openshift-apiserver] adding namespace took 5.458187ms I0425 23:02:13.334191 3778 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 5.599781ms I0425 23:02:13.334204 3778 namespace.go:98] [openshift-image-registry] adding namespace I0425 23:02:13.334743 3778 namespace.go:102] [openshift-infra] adding namespace took 5.791383ms I0425 23:02:13.335213 3778 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 5.501388ms I0425 23:02:13.335616 3778 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 5.179537ms I0425 23:02:13.336127 3778 namespace.go:102] [openshift-network-console] adding namespace took 4.832139ms I0425 23:02:13.336502 3778 namespace.go:102] [openshift-image-registry] adding namespace took 2.292303ms I0425 23:02:13.336696 3778 master.go:548] Adding or Updating local node "ip-10-0-2-186.ec2.internal" for network "default" I0425 23:02:13.337592 3778 zone_ic_handler.go:195] Time taken to create transit switch: 330.404µs I0425 23:02:13.337610 3778 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-217.ec2.internal for the network default I0425 23:02:13.338682 3778 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-217.ec2.internal" on network "default" took: 1.431449ms W0425 23:02:13.344945 3778 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-d6vnf: pod openshift-network-diagnostics/network-check-target-d6vnf: no pod IPs found W0425 23:02:13.344960 3778 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-6tvfc: pod openshift-multus/network-metrics-daemon-6tvfc: no pod IPs found I0425 23:02:13.345832 3778 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-186.ec2.internal for the network default I0425 23:02:13.345975 3778 zone_ic_handler.go:195] Time taken to create transit switch: 131.903µs I0425 23:02:13.346672 3778 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0425 23:02:13.346695 3778 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0425 23:02:13.346707 3778 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0425 23:02:13.346718 3778 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-217.ec2.internal in network "default" I0425 23:02:13.346773 3778 node_tracker.go:169] Node ip-10-0-1-217.ec2.internal switch + router changed, syncing services in network "default" I0425 23:02:13.346789 3778 services_controller.go:574] Full service sync requested for network=default I0425 23:02:13.346798 3778 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-186.ec2.internal in network "default" I0425 23:02:13.346817 3778 node_tracker.go:169] Node ip-10-0-2-186.ec2.internal switch + router changed, syncing services in network "default" I0425 23:02:13.346824 3778 services_controller.go:574] Full service sync requested for network=default I0425 23:02:13.447542 3778 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0425 23:02:13.447566 3778 services_controller.go:208] Setting up event handlers for services for network=default I0425 23:02:13.447687 3778 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0425 23:02:13.447740 3778 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0425 23:02:13.447769 3778 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0425 23:02:13.547983 3778 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0425 23:02:13.548004 3778 repair.go:60] Starting repairing loop for services I0425 23:02:13.548125 3778 repair.go:131] Deleted 0 stale service LBs I0425 23:02:13.548144 3778 repair.go:137] Deleted 0 stale Chassis Template Vars I0425 23:02:13.548159 3778 repair.go:62] Finished repairing loop for services: 157.491µs I0425 23:02:13.548183 3778 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0425 23:02:13.548193 3778 services_controller.go:254] Starting workers for network=default I0425 23:02:13.548524 3778 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-6tvfc] creating logical port openshift-multus_network-metrics-daemon-6tvfc for pod on switch ip-10-0-2-186.ec2.internal I0425 23:02:13.548584 3778 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-d6vnf] creating logical port openshift-network-diagnostics_network-check-target-d6vnf for pod on switch ip-10-0-2-186.ec2.internal I0425 23:02:13.548671 3778 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-6tvfc I0425 23:02:13.548670 3778 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-d6vnf I0425 23:02:13.565190 3778 pod.go:62] [openshift-network-diagnostics/network-check-target-d6vnf] pod update took 16.551611ms I0425 23:02:13.565209 3778 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-d6vnf] addLogicalPort annotation time took 16.57027ms I0425 23:02:13.565985 3778 pods.go:271] [openshift-network-diagnostics/network-check-target-d6vnf] addLogicalPort took 17.404148ms, libovsdb time 596.864µs I0425 23:02:13.566419 3778 pod.go:62] [openshift-multus/network-metrics-daemon-6tvfc] pod update took 17.807251ms I0425 23:02:13.566439 3778 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-6tvfc] addLogicalPort annotation time took 17.828792ms I0425 23:02:13.567185 3778 pods.go:271] [openshift-multus/network-metrics-daemon-6tvfc] addLogicalPort took 18.671524ms, libovsdb time 569.328µs I0425 23:02:13.567617 3778 admin_network_policy_controller.go:227] Starting controller default-network-controller I0425 23:02:13.567634 3778 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0425 23:02:13.567646 3778 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0425 23:02:13.567650 3778 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0425 23:02:13.567847 3778 repair.go:29] Repairing admin network policies took 191.91µs I0425 23:02:13.567954 3778 repair.go:92] Repairing baseline admin network policies took 98.105µs I0425 23:02:13.568418 3778 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0425 23:02:13.568442 3778 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0425 23:02:13.570444 3778 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0425 23:02:13.570470 3778 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0425 23:02:13.570485 3778 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0425 23:02:13.572014 3778 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0425 23:02:13.572032 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0425 23:02:13.572040 3778 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0425 23:02:13.573498 3778 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0425 23:02:13.573520 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0425 23:02:13.573532 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0425 23:02:13.574922 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0425 23:02:13.574941 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0425 23:02:13.574950 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0425 23:02:13.576523 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0425 23:02:13.576542 3778 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0425 23:02:13.576554 3778 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0425 23:02:13.577674 3778 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0425 23:02:13.577693 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0425 23:02:13.577702 3778 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0425 23:02:13.578188 3778 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0425 23:02:13.578206 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0425 23:02:13.578214 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0425 23:02:13.578670 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0425 23:02:13.578689 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0425 23:02:13.578697 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0425 23:02:13.579212 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0425 23:02:13.579233 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0425 23:02:13.579606 3778 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0425 23:02:13.579617 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0425 23:02:13.580279 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0425 23:02:13.580297 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0425 23:02:13.580304 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0425 23:02:13.580790 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0425 23:02:13.580808 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0425 23:02:13.580816 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0425 23:02:13.581313 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0425 23:02:13.581330 3778 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0425 23:02:13.581339 3778 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0425 23:02:13.582311 3778 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0425 23:02:13.582326 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0425 23:02:13.582348 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0425 23:02:13.582996 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0425 23:02:13.583018 3778 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0425 23:02:13.583027 3778 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0425 23:02:13.584530 3778 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0425 23:02:13.584546 3778 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0425 23:02:13.584555 3778 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0425 23:02:13.586426 3778 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0425 23:02:13.586809 3778 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0425 23:02:13.586818 3778 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0425 23:02:13.587634 3778 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0425 23:02:13.587652 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0425 23:02:13.587669 3778 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0425 23:02:13.588361 3778 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0425 23:02:13.588378 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0425 23:02:13.588385 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0425 23:02:13.588887 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0425 23:02:13.588906 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0425 23:02:13.588912 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0425 23:02:13.589191 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0425 23:02:13.589207 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0425 23:02:13.589216 3778 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0425 23:02:13.590418 3778 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0425 23:02:13.590436 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0425 23:02:13.590444 3778 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0425 23:02:13.590702 3778 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0425 23:02:13.590723 3778 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0425 23:02:13.590732 3778 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0425 23:02:13.591677 3778 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0425 23:02:13.591693 3778 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0425 23:02:13.591701 3778 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0425 23:02:13.592196 3778 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0425 23:02:13.592215 3778 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0425 23:02:13.592223 3778 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0425 23:02:13.593049 3778 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0425 23:02:13.593071 3778 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0425 23:02:13.593079 3778 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0425 23:02:13.593587 3778 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0425 23:02:13.593605 3778 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0425 23:02:13.593611 3778 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0425 23:02:13.593855 3778 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0425 23:02:13.593876 3778 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0425 23:02:13.593885 3778 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0425 23:02:13.595500 3778 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0425 23:02:13.595519 3778 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0425 23:02:13.595528 3778 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0425 23:02:13.595798 3778 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0425 23:02:13.600433 3778 egressfirewall.go:374] Starting EgressFirewall controller I0425 23:02:13.600468 3778 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0425 23:02:13.600467 3778 controller.go:133] Adding controller egress-firewall-controller event handlers I0425 23:02:13.600493 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0425 23:02:13.600504 3778 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0425 23:02:13.600515 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0425 23:02:13.600527 3778 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0425 23:02:13.600768 3778 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0425 23:02:13.600781 3778 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0425 23:02:13.600786 3778 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0425 23:02:13.600793 3778 egressqos.go:193] Setting up event handlers for EgressQoS I0425 23:02:13.600805 3778 egressfirewall.go:723] Syncing node "ip-10-0-1-217.ec2.internal" for egress firewall I0425 23:02:13.600818 3778 egressfirewall.go:723] Syncing node "ip-10-0-2-186.ec2.internal" for egress firewall I0425 23:02:13.600928 3778 egressqos.go:245] Starting EgressQoS Controller I0425 23:02:13.600940 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0425 23:02:13.600947 3778 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0425 23:02:13.600953 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0425 23:02:13.600958 3778 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0425 23:02:13.600963 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0425 23:02:13.600967 3778 shared_informer.go:356] "Caches are synced" controller="egressqos" I0425 23:02:13.600971 3778 egressqos.go:259] Repairing EgressQoSes I0425 23:02:13.600974 3778 egressqos.go:399] Starting repairing loop for egressqos I0425 23:02:13.601047 3778 egressqos.go:401] Finished repairing loop for egressqos: 72.084µs I0425 23:02:13.601064 3778 egressservice_zone.go:132] Setting up event handlers for Egress Services I0425 23:02:13.601084 3778 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-186.ec2.internal I0425 23:02:13.601133 3778 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-186.ec2.internal : 48.858µs I0425 23:02:13.601218 3778 egressservice_zone.go:210] Starting Egress Services Controller I0425 23:02:13.601232 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0425 23:02:13.601243 3778 shared_informer.go:356] "Caches are synced" controller="egressservices" I0425 23:02:13.601250 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0425 23:02:13.601258 3778 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0425 23:02:13.601266 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0425 23:02:13.601273 3778 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0425 23:02:13.601280 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0425 23:02:13.601285 3778 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0425 23:02:13.601289 3778 egressservice_zone.go:228] Repairing Egress Services I0425 23:02:13.601547 3778 master_controller.go:88] Starting Admin Policy Based Route Controller I0425 23:02:13.601556 3778 external_controller.go:278] Starting Admin Policy Based Route Controller I0425 23:02:13.601615 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-186.ec2.internal I0425 23:02:13.601627 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-186.ec2.internal: 12.751µs I0425 23:02:13.601636 3778 default_network_controller.go:585] Completing all the Watchers took 302.768734ms I0425 23:02:13.601638 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-217.ec2.internal I0425 23:02:13.601645 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-217.ec2.internal: 8.582µs I0425 23:02:13.601649 3778 default_network_controller.go:589] Starting unidling controllers I0425 23:02:13.601663 3778 unidle.go:46] Registering OVN SB ControllerEvent handler I0425 23:02:13.601668 3778 unidle.go:63] Populating Initial ContollerEvent events I0425 23:02:13.601678 3778 unidle.go:79] Setting up event handlers for services I0425 23:02:13.601717 3778 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0425 23:02:13.646717 3778 default_node_network_controller.go:519] Node connection status = connected I0425 23:02:14.146459 3778 default_node_network_controller.go:519] Node connection status = connected I0425 23:02:14.645959 3778 default_node_network_controller.go:519] Node connection status = connected I0425 23:02:15.146857 3778 default_node_network_controller.go:519] Node connection status = connected I0425 23:02:15.432144 3778 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0425 23:02:15.534909 3778 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-217.ec2.internal in network "default" I0425 23:02:15.534990 3778 node_tracker.go:169] Node ip-10-0-1-217.ec2.internal switch + router changed, syncing services in network "default" I0425 23:02:15.534995 3778 egressfirewall.go:723] Syncing node "ip-10-0-1-217.ec2.internal" for egress firewall I0425 23:02:15.535006 3778 services_controller.go:574] Full service sync requested for network=default I0425 23:02:15.535006 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-217.ec2.internal I0425 23:02:15.535021 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-217.ec2.internal: 21.498µs I0425 23:02:15.535073 3778 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-217.ec2.internal. Updating no re-route policies I0425 23:02:15.646107 3778 default_node_network_controller.go:519] Node connection status = connected I0425 23:02:15.665406 3778 bridgeconfig.go:537] Gateway is ready I0425 23:02:15.665426 3778 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0425 23:02:15.678376 3778 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.186. Updating OVN encap IP. I0425 23:02:15.684133 3778 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.186 - it is already configured I0425 23:02:15.684148 3778 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.186/24:{} 172.20.0.1/32:{}]. Updating annotations. I0425 23:02:15.684372 3778 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.186/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-186.ec2.internal","mac-address":"0e:97:be:5d:f0:43","ip-addresses":["10.0.2.186/24"],"ip-address":"10.0.2.186/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:2ed05171-23d6-4ed8-98cd-2bb1fbabb65b k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.186/24"}] on node ip-10-0-2-186.ec2.internal I0425 23:02:15.699895 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-186.ec2.internal I0425 23:02:15.699896 3778 master.go:548] Adding or Updating local node "ip-10-0-2-186.ec2.internal" for network "default" I0425 23:02:15.699916 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-186.ec2.internal: 25.941µs I0425 23:02:15.699928 3778 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-186.ec2.internal in network "default" I0425 23:02:15.699939 3778 egressfirewall.go:723] Syncing node "ip-10-0-2-186.ec2.internal" for egress firewall I0425 23:02:15.699987 3778 node_tracker.go:169] Node ip-10-0-2-186.ec2.internal switch + router changed, syncing services in network "default" I0425 23:02:15.700004 3778 services_controller.go:574] Full service sync requested for network=default I0425 23:02:15.700052 3778 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.186 I0425 23:02:15.700166 3778 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-186.ec2.internal I0425 23:02:15.704560 3778 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-186.ec2.internal. Updating no re-route policies I0425 23:02:15.704656 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-jbjws I0425 23:02:15.704679 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-jbjws I0425 23:02:15.704705 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-jbjws after 0 failed attempt(s) I0425 23:02:15.704711 3778 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-8rjwb I0425 23:02:15.704723 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-k45ph I0425 23:02:15.704726 3778 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-8rjwb I0425 23:02:15.704729 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-5nq9q I0425 23:02:15.704738 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-nvt5w I0425 23:02:15.704740 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-d6vnf I0425 23:02:15.704742 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-bc8tc I0425 23:02:15.704751 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-nvt5w I0425 23:02:15.704756 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-d6vnf I0425 23:02:15.704756 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-bc8tc I0425 23:02:15.704764 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-csxzw I0425 23:02:15.704765 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-45mjg I0425 23:02:15.704774 3778 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-8rjwb after 0 failed attempt(s) I0425 23:02:15.704777 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-bc8tc after 0 failed attempt(s) I0425 23:02:15.704770 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-6tvfc I0425 23:02:15.704784 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-45mjg I0425 23:02:15.704786 3778 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-d6vnf] creating logical port openshift-network-diagnostics_network-check-target-d6vnf for pod on switch ip-10-0-2-186.ec2.internal I0425 23:02:15.704789 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-6tvfc I0425 23:02:15.704746 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-5nq9q I0425 23:02:15.704792 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-nj6h2 I0425 23:02:15.704791 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-186.ec2.internal I0425 23:02:15.704797 3778 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-186.ec2.internal I0425 23:02:15.704802 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-nj6h2 I0425 23:02:15.704807 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-5nq9q after 0 failed attempt(s) I0425 23:02:15.704808 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-45mjg after 0 failed attempt(s) I0425 23:02:15.704815 3778 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-186.ec2.internal I0425 23:02:15.704806 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-186.ec2.internal I0425 23:02:15.704823 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-nj6h2 after 0 failed attempt(s) I0425 23:02:15.704825 3778 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-6tvfc] creating logical port openshift-multus_network-metrics-daemon-6tvfc for pod on switch ip-10-0-2-186.ec2.internal I0425 23:02:15.704834 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-186.ec2.internal after 0 failed attempt(s) I0425 23:02:15.704773 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-nvt5w after 0 failed attempt(s) I0425 23:02:15.704842 3778 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-186.ec2.internal after 0 failed attempt(s) I0425 23:02:15.704780 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-csxzw I0425 23:02:15.704861 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-csxzw after 0 failed attempt(s) I0425 23:02:15.704732 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-k45ph I0425 23:02:15.704896 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-k45ph after 0 failed attempt(s) I0425 23:02:15.705488 3778 pods.go:271] [openshift-network-diagnostics/network-check-target-d6vnf] addLogicalPort took 712.167µs, libovsdb time 255.6µs I0425 23:02:15.705502 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-d6vnf after 0 failed attempt(s) I0425 23:02:15.705648 3778 pods.go:271] [openshift-multus/network-metrics-daemon-6tvfc] addLogicalPort took 835.326µs, libovsdb time 375.007µs I0425 23:02:15.705656 3778 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-6tvfc after 0 failed attempt(s) I0425 23:02:15.717221 3778 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0425 23:02:15.730719 3778 iptables.go:108] Creating table: mangle chain: OUTPUT I0425 23:02:15.734548 3778 iptables.go:108] Creating table: nat chain: OUTPUT I0425 23:02:15.785214 3778 iptables.go:108] Creating table: nat chain: PREROUTING I0425 23:02:15.852504 3778 iptables.go:108] Creating table: filter chain: FORWARD I0425 23:02:15.871631 3778 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0425 23:02:15.871739 3778 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0425 23:02:15.871791 3778 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0425 23:02:15.871866 3778 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0425 23:02:15.871932 3778 gateway_init.go:325] Gateway and management port readiness took 2.730815033s I0425 23:02:15.871963 3778 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0425 23:02:15.872008 3778 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0425 23:02:15.872026 3778 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0425 23:02:15.872036 3778 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0425 23:02:15.872116 3778 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0425 23:02:15.872208 3778 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 47.864µs I0425 23:02:15.872448 3778 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0425 23:02:15.872480 3778 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 20.852µs I0425 23:02:15.872488 3778 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 42.339µs I0425 23:02:15.872495 3778 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0425 23:02:15.872508 3778 nad_controller.go:379] [node-nad-controller NAD controller]: started I0425 23:02:15.872514 3778 default_node_network_controller.go:899] Starting the default node network controller I0425 23:02:15.872518 3778 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0425 23:02:15.872608 3778 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 8.55µs I0425 23:02:15.872676 3778 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0425 23:02:15.872685 3778 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0425 23:02:15.872712 3778 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c12ce0 0xc0001c3f10 0xc0002ec000 0xc0002ec070 0xc0002ec0e0 0xc0002ec150 0xc0002ec1c0 0xc0007a3810 0xc0002ec230 0xc0002ec2a0 0xc000aac2d0 0xc00654d340 0xc0002ec380 map[0x2ff9080:0xc002813560 0x2ff95c0:0xc0028136e0 0x2ffa2e0:0xc00160e6c0 0x2ffa580:0xc002813500 0x2ffad60:0xc0028135c0 0x2ffb000:0xc002813620 0x2ffb2a0:0xc002813680 0x2ffba80:0xc002813740 0x2ffbd20:0xc0028139e0 0x3026980:0xc00160e420 0x3026d20:0xc00160e480 0x3027800:0xc000c7a600 0x3029160:0xc001b6c000 0x30298a0:0xc00160e660 0x302a720:0xc001f00480] 0xc00016a9a0 0} I0425 23:02:15.872784 3778 gateway.go:151] Starting gateway service sync I0425 23:02:15.873235 3778 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0425 23:02:15.875778 3778 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0425 23:02:15.900632 3778 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0425 23:02:15.902048 3778 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0425 23:02:15.903396 3778 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0425 23:02:15.939928 3778 gateway.go:168] Gateway service sync done. Time taken: 67.126958ms I0425 23:02:15.940422 3778 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31074 I0425 23:02:15.940766 3778 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c12ce0 0xc0001c3f10 0xc0002ec000 0xc0002ec070 0xc0002ec0e0 0xc0002ec150 0xc0002ec1c0 0xc0007a3810 0xc0002ec230 0xc0002ec2a0 0xc000aac2d0 0xc00654d340 0xc0002ec380 map[0x2ff9080:0xc002813560 0x2ff95c0:0xc0028136e0 0x2ffa2e0:0xc00160e6c0 0x2ffa580:0xc002813500 0x2ffad60:0xc0028135c0 0x2ffb000:0xc002813620 0x2ffb2a0:0xc002813680 0x2ffba80:0xc002813740 0x2ffbd20:0xc0028139e0 0x3026980:0xc00160e420 0x3026d20:0xc00160e480 0x3027800:0xc000c7a600 0x3029160:0xc001b6c000 0x30298a0:0xc00160e660 0x302a720:0xc001f00480] 0xc00016a9a0 0} I0425 23:02:15.941236 3778 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c12ce0 0xc0001c3f10 0xc0002ec000 0xc0002ec070 0xc0002ec0e0 0xc0002ec150 0xc0002ec1c0 0xc0007a3810 0xc0002ec230 0xc0002ec2a0 0xc000aac2d0 0xc00654d340 0xc0002ec380 map[0x2ff9080:0xc002813560 0x2ff95c0:0xc0028136e0 0x2ffa2e0:0xc00160e6c0 0x2ffa580:0xc002813500 0x2ffad60:0xc0028135c0 0x2ffb000:0xc002813620 0x2ffb2a0:0xc002813680 0x2ffba80:0xc002813740 0x2ffbd20:0xc0028139e0 0x3026980:0xc00160e420 0x3026d20:0xc00160e480 0x3027800:0xc000c7a600 0x3029160:0xc001b6c000 0x30298a0:0xc00160e660 0x302a720:0xc001f00480] 0xc00016a9a0 0} I0425 23:02:15.941278 3778 gateway.go:532] Reconciling gateway with updates I0425 23:02:15.947568 3778 gateway.go:339] Spawning Conntrack Rule Check Thread I0425 23:02:15.947632 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0425 23:02:15.947646 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0425 23:02:15.947654 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0425 23:02:15.947654 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0425 23:02:15.947659 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0425 23:02:15.947663 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0425 23:02:15.947671 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0425 23:02:15.947680 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0425 23:02:15.947679 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0425 23:02:15.947694 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0425 23:02:15.947705 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0425 23:02:15.947638 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0425 23:02:15.947710 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0425 23:02:15.947718 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0425 23:02:15.947723 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0425 23:02:15.947726 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0425 23:02:15.947726 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0425 23:02:15.947732 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0425 23:02:15.947737 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0425 23:02:15.947661 3778 default_node_network_controller.go:944] Gateway and management port readiness took 74.96838ms I0425 23:02:15.947739 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0425 23:02:15.947748 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0425 23:02:15.947748 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0425 23:02:15.947752 3778 port_claim.go:71] Svc openshift-ingress/router-default: port 30286 is already open, no action needed I0425 23:02:15.947760 3778 port_claim.go:71] Svc openshift-ingress/router-default: port 31459 is already open, no action needed I0425 23:02:15.947782 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0425 23:02:15.947733 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0425 23:02:15.947804 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 23:02:15.947818 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 23:02:15.947712 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0425 23:02:15.948202 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0425 23:02:15.948215 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0425 23:02:15.948222 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0425 23:02:15.948228 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0425 23:02:15.948233 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0425 23:02:15.948239 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0425 23:02:15.948247 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0425 23:02:15.948250 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0425 23:02:15.948256 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0425 23:02:15.948260 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0425 23:02:15.948212 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0425 23:02:15.948272 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0425 23:02:15.948242 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0425 23:02:15.948288 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0425 23:02:15.948293 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0425 23:02:15.948303 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0425 23:02:15.948312 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0425 23:02:15.948321 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0425 23:02:15.948324 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0425 23:02:15.948329 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0425 23:02:15.948331 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0425 23:02:15.948329 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0425 23:02:15.948336 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0425 23:02:15.948338 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0425 23:02:15.948338 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0425 23:02:15.948353 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0425 23:02:15.948409 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0425 23:02:15.948426 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0425 23:02:15.948434 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0425 23:02:15.948438 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0425 23:02:15.948449 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0425 23:02:15.948451 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0425 23:02:15.948453 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0425 23:02:15.948461 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0425 23:02:15.948471 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0425 23:02:15.948474 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0425 23:02:15.948490 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0425 23:02:15.948491 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0425 23:02:15.948492 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0425 23:02:15.948500 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0425 23:02:15.948500 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0425 23:02:15.948509 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0425 23:02:15.948525 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0425 23:02:15.948536 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0425 23:02:15.948539 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0425 23:02:15.948544 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0425 23:02:15.948543 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0425 23:02:15.948548 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0425 23:02:15.948555 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0425 23:02:15.948589 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0425 23:02:15.948598 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0425 23:02:15.948600 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0425 23:02:15.948610 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0425 23:02:15.948647 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0425 23:02:15.948673 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0425 23:02:15.948683 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0425 23:02:15.948693 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0425 23:02:15.948694 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0425 23:02:15.948700 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0425 23:02:15.948705 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0425 23:02:15.948707 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0425 23:02:15.948719 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0425 23:02:15.948720 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0425 23:02:15.948723 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0425 23:02:15.948727 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0425 23:02:15.948732 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0425 23:02:15.948732 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0425 23:02:15.948739 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0425 23:02:15.948747 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0425 23:02:15.948741 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0425 23:02:15.948760 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0425 23:02:15.948762 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0425 23:02:15.948763 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0425 23:02:15.948779 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0425 23:02:15.948793 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0425 23:02:15.948722 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0425 23:02:15.948846 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0425 23:02:15.949380 3778 node_ip_handler_linux.go:160] Node IP manager is running W0425 23:02:15.988007 3778 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0425 23:02:15.988223 3778 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0425 23:02:15.988303 3778 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0425 23:02:15.997568 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0425 23:02:15.997575 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0425 23:02:15.997591 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0425 23:02:15.997598 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0425 23:02:15.997605 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0425 23:02:15.997610 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0425 23:02:15.997616 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0425 23:02:16.040767 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0425 23:02:16.040806 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0425 23:02:16.040817 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0425 23:02:16.082148 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0425 23:02:16.082177 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0425 23:02:16.082183 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0425 23:02:16.082189 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0425 23:02:16.082195 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0425 23:02:16.082200 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0425 23:02:16.082206 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0425 23:02:16.082212 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0425 23:02:16.082216 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0425 23:02:16.082224 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0425 23:02:16.082229 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0425 23:02:16.082236 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0425 23:02:16.082241 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0425 23:02:16.082246 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0425 23:02:16.082251 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0425 23:02:16.082256 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0425 23:02:16.082260 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0425 23:02:16.082266 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0425 23:02:17.731624 3778 default_node_network_controller.go:1380] Starting node controller node sync W0425 23:02:17.731672 3778 egressip_healthcheck.go:76] Health checking using insecure connection I0425 23:02:17.731720 3778 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0425 23:02:17.743452 3778 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.805653ms I0425 23:02:17.743579 3778 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-217.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0425 23:02:17.743597 3778 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-217.ec2.internal", IP: 10.0.1.217 to PMTUD blocking rules I0425 23:02:17.749657 3778 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0425 23:02:17.749981 3778 egressservice_node.go:93] Setting up event handlers for Egress Services I0425 23:02:17.750112 3778 egressservice_node.go:183] Starting Egress Services Controller I0425 23:02:17.750129 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0425 23:02:17.750139 3778 shared_informer.go:356] "Caches are synced" controller="egressservices" I0425 23:02:17.750145 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0425 23:02:17.750151 3778 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0425 23:02:17.750158 3778 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0425 23:02:17.750162 3778 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0425 23:02:17.750167 3778 egressservice_node.go:197] Repairing Egress Services I0425 23:02:17.760430 3778 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0425 23:02:17.760441 3778 external_controller.go:278] Starting Admin Policy Based Route Controller I0425 23:02:17.760573 3778 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0425 23:02:17.760583 3778 link_network_manager.go:119] Link manager is running I0425 23:02:17.760591 3778 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0425 23:02:17.760805 3778 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0425 23:02:17.761023 3778 vrf_manager.go:128] VRF manager is running I0425 23:02:17.761332 3778 node_controller_manager.go:437] Removing flows to drop GARP I0425 23:02:17.761345 3778 gateway.go:532] Reconciling gateway with updates I0425 23:02:17.776958 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0425 23:02:17.776960 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0425 23:02:17.776981 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0425 23:02:17.776990 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0425 23:02:17.776994 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0425 23:02:17.777002 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0425 23:02:17.777009 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0425 23:02:17.777010 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0425 23:02:17.777013 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0425 23:02:17.777022 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0425 23:02:17.777022 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0425 23:02:17.777028 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0425 23:02:17.777020 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0425 23:02:17.777033 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0425 23:02:17.777045 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0425 23:02:17.777048 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0425 23:02:17.777054 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0425 23:02:17.777054 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0425 23:02:17.777064 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0425 23:02:17.777050 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0425 23:02:17.777069 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0425 23:02:17.777075 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0425 23:02:17.777083 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0425 23:02:17.777085 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0425 23:02:17.777108 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0425 23:02:17.777110 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0425 23:02:17.777090 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0425 23:02:17.777115 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0425 23:02:17.777113 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0425 23:02:17.777125 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0425 23:02:17.777126 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0425 23:02:17.777129 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0425 23:02:17.777115 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 23:02:17.777133 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0425 23:02:17.777141 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0425 23:02:17.777138 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0425 23:02:17.777151 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0425 23:02:17.777149 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0425 23:02:17.777077 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0425 23:02:17.777160 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0425 23:02:17.777171 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0425 23:02:17.777173 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0425 23:02:17.777180 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0425 23:02:17.777183 3778 port_claim.go:71] Svc openshift-ingress/router-default: port 30286 is already open, no action needed I0425 23:02:17.777185 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0425 23:02:17.777190 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0425 23:02:17.777191 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0425 23:02:17.777202 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0425 23:02:17.777203 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 23:02:17.777203 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0425 23:02:17.777217 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0425 23:02:17.777214 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0425 23:02:17.777213 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0425 23:02:17.777222 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0425 23:02:17.777223 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0425 23:02:17.777232 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0425 23:02:17.777234 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0425 23:02:17.777236 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0425 23:02:17.777237 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0425 23:02:17.777237 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0425 23:02:17.777246 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0425 23:02:17.777124 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0425 23:02:17.777251 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0425 23:02:17.777258 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0425 23:02:17.777263 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0425 23:02:17.777241 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0425 23:02:17.777269 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0425 23:02:17.777125 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0425 23:02:17.777173 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0425 23:02:17.777290 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0425 23:02:17.777165 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0425 23:02:17.777302 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0425 23:02:17.777248 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0425 23:02:17.777153 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0425 23:02:17.777234 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0425 23:02:17.777353 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0425 23:02:17.777119 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0425 23:02:17.777378 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0425 23:02:17.777178 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0425 23:02:17.777398 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0425 23:02:17.777411 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0425 23:02:17.777177 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0425 23:02:17.777175 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0425 23:02:17.777427 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0425 23:02:17.777432 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0425 23:02:17.777437 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0425 23:02:17.777444 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0425 23:02:17.777430 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0425 23:02:17.777139 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0425 23:02:17.777498 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0425 23:02:17.777220 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0425 23:02:17.777223 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0425 23:02:17.777536 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0425 23:02:17.777541 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0425 23:02:17.777220 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0425 23:02:17.777547 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0425 23:02:17.777231 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0425 23:02:17.777571 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0425 23:02:17.777229 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0425 23:02:17.777250 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0425 23:02:17.777661 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0425 23:02:17.777243 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0425 23:02:17.777276 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0425 23:02:17.777160 3778 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0425 23:02:17.777699 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0425 23:02:17.777421 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0425 23:02:17.777183 3778 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0425 23:02:17.777200 3778 port_claim.go:71] Svc openshift-ingress/router-default: port 31459 is already open, no action needed I0425 23:02:17.801649 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0425 23:02:17.801670 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0425 23:02:17.801679 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0425 23:02:17.801684 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0425 23:02:17.801690 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0425 23:02:17.801696 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0425 23:02:17.841049 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0425 23:02:17.841072 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0425 23:02:17.841079 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0425 23:02:17.841084 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0425 23:02:17.841108 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0425 23:02:17.841116 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0425 23:02:17.841123 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0425 23:02:17.841128 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0425 23:02:17.841132 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0425 23:02:17.841137 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0425 23:02:17.841144 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0425 23:02:17.841149 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0425 23:02:17.841154 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0425 23:02:17.841161 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0425 23:02:17.841171 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0425 23:02:17.841177 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0425 23:02:17.841185 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0425 23:02:17.841190 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0425 23:02:17.841195 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0425 23:02:17.879958 3778 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0425 23:02:19.260475 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-186.ec2.internal I0425 23:02:19.260504 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-186.ec2.internal: 36.12µs I0425 23:02:19.285367 3778 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6b57c9b684-qnb86] creating logical port openshift-image-registry_image-registry-6b57c9b684-qnb86 for pod on switch ip-10-0-2-186.ec2.internal I0425 23:02:19.285477 3778 kube.go:256] Updating pod openshift-image-registry/image-registry-6b57c9b684-qnb86 I0425 23:02:19.309924 3778 pod.go:62] [openshift-image-registry/image-registry-6b57c9b684-qnb86] pod update took 24.475143ms I0425 23:02:19.309944 3778 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6b57c9b684-qnb86] addLogicalPort annotation time took 24.501028ms I0425 23:02:19.311235 3778 pods.go:271] [openshift-image-registry/image-registry-6b57c9b684-qnb86] addLogicalPort took 25.886904ms, libovsdb time 924.953µs I0425 23:02:19.326459 3778 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-vzr5r] creating logical port openshift-dns_dns-default-vzr5r for pod on switch ip-10-0-2-186.ec2.internal I0425 23:02:19.326500 3778 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-svcvh] creating logical port openshift-ingress-canary_ingress-canary-svcvh for pod on switch ip-10-0-2-186.ec2.internal I0425 23:02:19.326574 3778 kube.go:256] Updating pod openshift-dns/dns-default-vzr5r I0425 23:02:19.326619 3778 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-svcvh I0425 23:02:19.326661 3778 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29619300-zwscc for pod on switch ip-10-0-2-186.ec2.internal I0425 23:02:19.326755 3778 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc I0425 23:02:19.327214 3778 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-dfp8s] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-dfp8s for pod on switch ip-10-0-2-186.ec2.internal I0425 23:02:19.327278 3778 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-dfp8s I0425 23:02:19.343773 3778 pod.go:62] [openshift-ingress-canary/ingress-canary-svcvh] pod update took 17.172773ms I0425 23:02:19.343791 3778 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-svcvh] addLogicalPort annotation time took 17.193302ms I0425 23:02:19.344756 3778 pods.go:271] [openshift-ingress-canary/ingress-canary-svcvh] addLogicalPort took 18.271597ms, libovsdb time 734.538µs I0425 23:02:19.345478 3778 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc] pod update took 18.74276ms I0425 23:02:19.345487 3778 pod.go:62] [openshift-dns/dns-default-vzr5r] pod update took 18.936275ms I0425 23:02:19.345495 3778 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc] addLogicalPort annotation time took 18.762424ms I0425 23:02:19.345499 3778 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-vzr5r] addLogicalPort annotation time took 18.95161ms I0425 23:02:19.346538 3778 pods.go:271] [openshift-dns/dns-default-vzr5r] addLogicalPort took 20.095003ms, libovsdb time 740.103µs I0425 23:02:19.347008 3778 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc] addLogicalPort took 20.357982ms, libovsdb time 1.124015ms I0425 23:02:19.352276 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-217.ec2.internal I0425 23:02:19.352300 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-217.ec2.internal: 30.504µs I0425 23:02:19.353961 3778 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-dfp8s] pod update took 26.697065ms I0425 23:02:19.353993 3778 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-dfp8s] addLogicalPort annotation time took 26.726719ms I0425 23:02:19.354923 3778 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-dfp8s] addLogicalPort took 27.714217ms, libovsdb time 683.75µs I0425 23:02:19.739661 3778 cni.go:452] [openshift-ingress-canary/ingress-canary-svcvh 650586e476bff1185f05e7cf9d7dcb7039c48fa5a3bd57e8b21fff78fa1933bb network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-svcvh 650586e476bff1185f05e7cf9d7dcb7039c48fa5a3bd57e8b21fff78fa1933bb network default NAD default NAD key ] I0425 23:02:19.776162 3778 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc 74eccb0863bc8d346f697ad679eb7db5a37a96699b16dce1a8f3592fc021e34b network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc 74eccb0863bc8d346f697ad679eb7db5a37a96699b16dce1a8f3592fc021e34b network default NAD default NAD key ] I0425 23:02:19.781063 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-svcvh, hostIfaceName: 650586e476bff11, network: default, NAD default, SandboxID: "650586e476bff1185f05e7cf9d7dcb7039c48fa5a3bd57e8b21fff78fa1933bb", PCI device ID: , UID: "90e8681b-f13e-44b7-b510-19afd991cf4e", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0425 23:02:19.786491 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29619300-zwscc, hostIfaceName: 74eccb0863bc8d3, network: default, NAD default, SandboxID: "74eccb0863bc8d346f697ad679eb7db5a37a96699b16dce1a8f3592fc021e34b", PCI device ID: , UID: "40ded3cb-7394-41e3-869d-25be992514eb", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0425 23:02:19.793426 3778 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-dfp8s c733b83d05b4cce9f06d2469a714fbbf60a06ce921c96052dc75c20d7560c4db network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-dfp8s c733b83d05b4cce9f06d2469a714fbbf60a06ce921c96052dc75c20d7560c4db network default NAD default NAD key ] I0425 23:02:19.804323 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-dfp8s, hostIfaceName: c733b83d05b4cce, network: default, NAD default, SandboxID: "c733b83d05b4cce9f06d2469a714fbbf60a06ce921c96052dc75c20d7560c4db", PCI device ID: , UID: "7de075be-83c8-4b0a-aa18-6972675d07f1", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0425 23:02:19.835395 3778 cni.go:473] [openshift-ingress-canary/ingress-canary-svcvh 650586e476bff1185f05e7cf9d7dcb7039c48fa5a3bd57e8b21fff78fa1933bb network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-svcvh 650586e476bff1185f05e7cf9d7dcb7039c48fa5a3bd57e8b21fff78fa1933bb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:0e:37:9b:25:b5\",\"name\":\"650586e476bff11\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9ee6d4b5-bb50-4462-b45d-c6835d42a4de\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:02:19.841293 3778 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc 74eccb0863bc8d346f697ad679eb7db5a37a96699b16dce1a8f3592fc021e34b network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc 74eccb0863bc8d346f697ad679eb7db5a37a96699b16dce1a8f3592fc021e34b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:fd:26:2f:1a:dd\",\"name\":\"74eccb0863bc8d3\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5b73f6ab-e976-445c-b33c-5f1030d2f59a\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:02:19.852406 3778 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-dfp8s c733b83d05b4cce9f06d2469a714fbbf60a06ce921c96052dc75c20d7560c4db network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-dfp8s c733b83d05b4cce9f06d2469a714fbbf60a06ce921c96052dc75c20d7560c4db network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:c2:46:19:a4:75\",\"name\":\"c733b83d05b4cce\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cb11b676-be62-435d-a04f-9d1fd0952ac4\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:02:28.161754 3778 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 23:02:30.073149 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-165.ec2.internal I0425 23:02:30.073174 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-165.ec2.internal: 32.096µs I0425 23:02:30.073158 3778 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-165.ec2.internal in network "default" I0425 23:02:30.073181 3778 egressfirewall.go:723] Syncing node "ip-10-0-0-165.ec2.internal" for egress firewall I0425 23:02:30.073192 3778 node_tracker.go:220] Node ip-10-0-0-165.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation E0425 23:02:30.073195 3778 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-165.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0425 23:02:30.073216 3778 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-165.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-165.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-165.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-165.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-165.ec2.internal, error parsing annotation for node ip-10-0-0-165.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0425 23:02:30.091783 3778 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-165.ec2.internal annotation: unexpected end of JSON input E0425 23:02:30.091807 3778 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-165.ec2.internal annotation: unexpected end of JSON input I0425 23:02:30.091833 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0425 23:02:30.091901 3778 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-165.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-165.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-165.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-165.ec2.internal" I0425 23:02:30.104135 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0425 23:02:30.104200 3778 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-165.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-165.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-165.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-165.ec2.internal" I0425 23:02:30.116700 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0425 23:02:30.129864 3778 egressfirewall.go:723] Syncing node "ip-10-0-0-165.ec2.internal" for egress firewall I0425 23:02:30.129915 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-165.ec2.internal I0425 23:02:30.129930 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-165.ec2.internal: 16.767µs I0425 23:02:30.129976 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 23:02:30.341851 3778 egressfirewall.go:723] Syncing node "ip-10-0-0-165.ec2.internal" for egress firewall I0425 23:02:30.341904 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-165.ec2.internal I0425 23:02:30.341912 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-165.ec2.internal: 10.091µs I0425 23:02:30.341958 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 23:02:30.354651 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 23:02:30.549608 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 23:02:30.615144 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 23:02:40.145830 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0425 23:02:41.189603 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 23:02:43.347332 3778 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-0-165.ec2.internal I0425 23:02:43.347361 3778 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-0-165.ec2.internal I0425 23:02:43.347458 3778 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-0-165.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-165.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-165.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-165.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-165.ec2.internal I0425 23:02:45.706362 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-724h6 I0425 23:02:45.706388 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-724h6 I0425 23:02:45.706428 3778 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-724h6, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-724h6: suppressed error logged: pod openshift-network-diagnostics/network-check-target-724h6: no pod IPs found I0425 23:02:45.706435 3778 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-smptr I0425 23:02:45.706452 3778 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-smptr I0425 23:02:45.706485 3778 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-smptr, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-smptr: suppressed error logged: pod openshift-multus/network-metrics-daemon-smptr: no pod IPs found I0425 23:02:47.997748 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-165.ec2.internal I0425 23:02:47.997771 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-165.ec2.internal: 29.66µs I0425 23:02:47.997791 3778 egressfirewall.go:723] Syncing node "ip-10-0-0-165.ec2.internal" for egress firewall I0425 23:02:47.997800 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 23:02:52.220412 3778 cni.go:452] [openshift-network-diagnostics/network-check-target-d6vnf 53f96d5158d7dc89e4c73fd61dabab58bc6f8b9d3ff9af588740b34d33194407 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-d6vnf 53f96d5158d7dc89e4c73fd61dabab58bc6f8b9d3ff9af588740b34d33194407 network default NAD default NAD key ] I0425 23:02:52.231130 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-d6vnf, hostIfaceName: 53f96d5158d7dc8, network: default, NAD default, SandboxID: "53f96d5158d7dc89e4c73fd61dabab58bc6f8b9d3ff9af588740b34d33194407", PCI device ID: , UID: "43d930c3-4485-4651-9a6a-98a78d723cf2", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0425 23:02:52.277323 3778 cni.go:473] [openshift-network-diagnostics/network-check-target-d6vnf 53f96d5158d7dc89e4c73fd61dabab58bc6f8b9d3ff9af588740b34d33194407 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-d6vnf 53f96d5158d7dc89e4c73fd61dabab58bc6f8b9d3ff9af588740b34d33194407 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:b4:3b:b3:d2:6d\",\"name\":\"53f96d5158d7dc8\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/838cfc75-7ded-48b6-a497-1a021c18e506\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:02:52.962565 3778 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-165.ec2.internal in network "default" I0425 23:02:52.962591 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "ip-10-0-0-165.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 23:02:52.962615 3778 node_tracker.go:235] Node ip-10-0-0-165.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-165.ec2.internal" I0425 23:02:52.962631 3778 node_tracker.go:169] Node ip-10-0-0-165.ec2.internal switch + router changed, syncing services in network "default" I0425 23:02:52.962644 3778 services_controller.go:574] Full service sync requested for network=default I0425 23:02:53.096753 3778 default_network_controller.go:1025] Node "ip-10-0-0-165.ec2.internal" in remote zone "ip-10-0-0-165.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 23:02:53.096777 3778 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-165.ec2.internal in network "default" I0425 23:02:53.096910 3778 node_tracker.go:169] Node ip-10-0-0-165.ec2.internal switch + router changed, syncing services in network "default" I0425 23:02:53.096925 3778 services_controller.go:574] Full service sync requested for network=default I0425 23:02:53.097889 3778 zone_ic_handler.go:195] Time taken to create transit switch: 310.279µs I0425 23:02:53.097907 3778 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-165.ec2.internal for the network default I0425 23:02:53.098891 3778 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-165.ec2.internal" on network "default" took: 1.322446ms W0425 23:02:55.227744 3778 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0425 23:02:55.654128 3778 egressfirewall.go:723] Syncing node "ip-10-0-0-165.ec2.internal" for egress firewall I0425 23:02:55.654128 3778 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-165.ec2.internal in network "default" I0425 23:02:55.654148 3778 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-165.ec2.internal", IP: 10.0.0.165 to PMTUD blocking rules I0425 23:02:55.654160 3778 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-165.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0425 23:02:55.654204 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-165.ec2.internal I0425 23:02:55.654212 3778 node_tracker.go:169] Node ip-10-0-0-165.ec2.internal switch + router changed, syncing services in network "default" I0425 23:02:55.654220 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-165.ec2.internal: 20.253µs I0425 23:02:55.654226 3778 services_controller.go:574] Full service sync requested for network=default W0425 23:02:55.654551 3778 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0425 23:02:55.660317 3778 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-165.ec2.internal. Updating no re-route policies I0425 23:02:55.668876 3778 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-165.ec2.internal", IP: 10.0.0.165 to PMTUD blocking rules I0425 23:02:55.668892 3778 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-165.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0425 23:02:55.720381 3778 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0425 23:02:59.423070 3778 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-165.ec2.internal I0425 23:02:59.423122 3778 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-165.ec2.internal: 59.057µs W0425 23:03:07.158612 3778 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0425 23:03:26.235488 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0425 23:04:23.750630 3778 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-8668968b4f-x7pdx] creating logical port openshift-ingress_router-default-8668968b4f-x7pdx for pod on switch ip-10-0-2-186.ec2.internal I0425 23:04:23.750748 3778 kube.go:256] Updating pod openshift-ingress/router-default-8668968b4f-x7pdx I0425 23:04:23.755828 3778 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-85cdb9646-dv94b] creating logical port openshift-ingress_router-default-85cdb9646-dv94b for pod on switch ip-10-0-2-186.ec2.internal I0425 23:04:23.755963 3778 kube.go:256] Updating pod openshift-ingress/router-default-85cdb9646-dv94b I0425 23:04:23.774363 3778 pod.go:62] [openshift-ingress/router-default-85cdb9646-dv94b] pod update took 18.439774ms I0425 23:04:23.774385 3778 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-85cdb9646-dv94b] addLogicalPort annotation time took 18.46603ms I0425 23:04:23.774515 3778 pod.go:62] [openshift-ingress/router-default-8668968b4f-x7pdx] pod update took 23.808113ms I0425 23:04:23.774538 3778 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-8668968b4f-x7pdx] addLogicalPort annotation time took 23.834254ms I0425 23:04:23.775643 3778 pods.go:271] [openshift-ingress/router-default-85cdb9646-dv94b] addLogicalPort took 19.833076ms, libovsdb time 955.228µs I0425 23:04:23.775948 3778 pods.go:271] [openshift-ingress/router-default-8668968b4f-x7pdx] addLogicalPort took 25.328488ms, libovsdb time 1.158512ms W0425 23:04:26.923792 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0425 23:04:28.289009 3778 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0425 23:04:28.289958 3778 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 921.198µs I0425 23:04:28.311673 3778 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0425 23:04:29.275184 3778 namespace.go:98] [openshift-service-ca] adding namespace I0425 23:04:29.276028 3778 namespace.go:102] [openshift-service-ca] adding namespace took 816.782µs I0425 23:04:29.290898 3778 namespace.go:142] [openshift-service-ca] updating namespace W0425 23:04:38.902937 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0425 23:04:43.502767 3778 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlt4s6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:04:43.503378 3778 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlt4s6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:04:43.503416 3778 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlt4s6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:04:43.503428 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0425 23:04:43.503444 3778 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlt4s6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:04:43.503453 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0425 23:04:54.325232 3778 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-djxcn] creating logical port openshift-insights_insights-runtime-extractor-djxcn for pod on switch ip-10-0-2-186.ec2.internal I0425 23:04:54.325316 3778 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-djxcn I0425 23:04:54.327908 3778 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlt4s6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:04:54.327938 3778 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlt4s6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:04:54.327954 3778 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlt4s6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:04:54.327960 3778 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlt4s6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:04:54.338673 3778 pod.go:62] [openshift-insights/insights-runtime-extractor-djxcn] pod update took 13.38306ms I0425 23:04:54.338696 3778 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-djxcn] addLogicalPort annotation time took 13.410256ms I0425 23:04:54.339832 3778 pods.go:271] [openshift-insights/insights-runtime-extractor-djxcn] addLogicalPort took 14.615561ms, libovsdb time 858.617µs I0425 23:04:54.684942 3778 cni.go:452] [openshift-insights/insights-runtime-extractor-djxcn 1168aacebb58e0c95bf33696bc0bbb31f013c1fe51de3bb50135bee71df5a5e6 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-djxcn 1168aacebb58e0c95bf33696bc0bbb31f013c1fe51de3bb50135bee71df5a5e6 network default NAD default NAD key ] I0425 23:04:54.695895 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-djxcn, hostIfaceName: 1168aacebb58e0c, network: default, NAD default, SandboxID: "1168aacebb58e0c95bf33696bc0bbb31f013c1fe51de3bb50135bee71df5a5e6", PCI device ID: , UID: "a69fe61c-10d9-4168-b5a8-cc4deb20b521", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0425 23:04:54.750089 3778 cni.go:473] [openshift-insights/insights-runtime-extractor-djxcn 1168aacebb58e0c95bf33696bc0bbb31f013c1fe51de3bb50135bee71df5a5e6 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-djxcn 1168aacebb58e0c95bf33696bc0bbb31f013c1fe51de3bb50135bee71df5a5e6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:41:80:93:ed:a6\",\"name\":\"1168aacebb58e0c\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7b0cf036-cf35-4d64-a65e-ba1bf34a63f5\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:04:55.427341 3778 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7c4f69cb6c-ps4xr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:04:55.428022 3778 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7c4f69cb6c-ps4xr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:04:55.443691 3778 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7c4f69cb6c-ps4xr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:04:55.443711 3778 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7c4f69cb6c-ps4xr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:04:55.447870 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c4f69cb6c-ps4xr I0425 23:04:55.447891 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c4f69cb6c-ps4xr I0425 23:04:55.505702 3778 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6b57c9b684-qnb86 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:04:55.505734 3778 pods.go:176] Deleting pod: openshift-image-registry/image-registry-6b57c9b684-qnb86 I0425 23:04:55.506924 3778 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-6b57c9b684-qnb86, ips: 10.128.0.5 I0425 23:04:55.506952 3778 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6b57c9b684-qnb86 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:04:55.512791 3778 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6466bdf7dd-z2xlz] creating logical port openshift-image-registry_image-registry-6466bdf7dd-z2xlz for pod on switch ip-10-0-2-186.ec2.internal I0425 23:04:55.512922 3778 kube.go:256] Updating pod openshift-image-registry/image-registry-6466bdf7dd-z2xlz I0425 23:04:55.520517 3778 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6b57c9b684-qnb86 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:04:55.520539 3778 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6b57c9b684-qnb86 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:04:55.523518 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6b57c9b684-qnb86 I0425 23:04:55.523546 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6b57c9b684-qnb86 I0425 23:04:55.532647 3778 pod.go:62] [openshift-image-registry/image-registry-6466bdf7dd-z2xlz] pod update took 19.764411ms I0425 23:04:55.532678 3778 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6466bdf7dd-z2xlz] addLogicalPort annotation time took 19.799321ms I0425 23:04:55.534088 3778 pods.go:271] [openshift-image-registry/image-registry-6466bdf7dd-z2xlz] addLogicalPort took 21.31948ms, libovsdb time 1.020437ms I0425 23:04:55.875796 3778 cni.go:452] [openshift-image-registry/image-registry-6466bdf7dd-z2xlz 4a33337a6df8b8aeee5c3c515b4fce64193d776d1f0f08d455271ef07baf1c22 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6466bdf7dd-z2xlz 4a33337a6df8b8aeee5c3c515b4fce64193d776d1f0f08d455271ef07baf1c22 network default NAD default NAD key ] I0425 23:04:55.886861 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6466bdf7dd-z2xlz, hostIfaceName: 4a33337a6df8b8a, network: default, NAD default, SandboxID: "4a33337a6df8b8aeee5c3c515b4fce64193d776d1f0f08d455271ef07baf1c22", PCI device ID: , UID: "5f1ec7f2-5dd9-49b3-9a76-a9e028100288", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0425 23:04:55.917581 3778 cni.go:452] [openshift-ingress/router-default-85cdb9646-dv94b 5568cc7eea241413657283ce641858f4ae397ba57b366f1fad9e642c41657bda network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-85cdb9646-dv94b 5568cc7eea241413657283ce641858f4ae397ba57b366f1fad9e642c41657bda network default NAD default NAD key ] I0425 23:04:55.920721 3778 cni.go:452] [openshift-ingress/router-default-8668968b4f-x7pdx fb84f99e8dcb2a3f0a0f21061715ba7bbf016f83acc464fad15191fbe4a851fc network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-8668968b4f-x7pdx fb84f99e8dcb2a3f0a0f21061715ba7bbf016f83acc464fad15191fbe4a851fc network default NAD default NAD key ] I0425 23:04:55.934567 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-8668968b4f-x7pdx, hostIfaceName: fb84f99e8dcb2a3, network: default, NAD default, SandboxID: "fb84f99e8dcb2a3f0a0f21061715ba7bbf016f83acc464fad15191fbe4a851fc", PCI device ID: , UID: "ba3aa69a-f1c3-4ff0-8fa4-0aca1901467c", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0425 23:04:55.934734 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-85cdb9646-dv94b, hostIfaceName: 5568cc7eea24141, network: default, NAD default, SandboxID: "5568cc7eea241413657283ce641858f4ae397ba57b366f1fad9e642c41657bda", PCI device ID: , UID: "e092288a-6208-4815-a8ed-1f0085d3cbea", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0425 23:04:55.939644 3778 cni.go:473] [openshift-image-registry/image-registry-6466bdf7dd-z2xlz 4a33337a6df8b8aeee5c3c515b4fce64193d776d1f0f08d455271ef07baf1c22 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6466bdf7dd-z2xlz 4a33337a6df8b8aeee5c3c515b4fce64193d776d1f0f08d455271ef07baf1c22 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:0b:c0:45:d5:6e\",\"name\":\"4a33337a6df8b8a\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3e96f2fc-9be1-43f5-8c4c-93e7c8bb5b87\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:04:55.984889 3778 cni.go:473] [openshift-ingress/router-default-85cdb9646-dv94b 5568cc7eea241413657283ce641858f4ae397ba57b366f1fad9e642c41657bda network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-85cdb9646-dv94b 5568cc7eea241413657283ce641858f4ae397ba57b366f1fad9e642c41657bda network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:01:e6:6e:f2:ba\",\"name\":\"5568cc7eea24141\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/064610b4-1198-4274-8875-9c57be1d6eb4\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:04:56.003012 3778 cni.go:473] [openshift-ingress/router-default-8668968b4f-x7pdx fb84f99e8dcb2a3f0a0f21061715ba7bbf016f83acc464fad15191fbe4a851fc network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-8668968b4f-x7pdx fb84f99e8dcb2a3f0a0f21061715ba7bbf016f83acc464fad15191fbe4a851fc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:cb:6d:bb:59:1c\",\"name\":\"fb84f99e8dcb2a3\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d0501ace-a023-4f74-bf80-fe01325a7b3f\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:04:56.442425 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:56.442521 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:56.444356 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:56.444375 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:56.444383 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:56.465692 3778 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0425 23:04:56.514880 3778 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0425 23:04:56.543548 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:56.543580 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:56.543623 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:56.543639 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:56.543647 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:56.564679 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:56.564741 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:56.564762 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:56.564859 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:56.564876 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0425 23:04:56.954379 3778 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0425 23:04:56.960629 3778 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0425 23:04:57.502113 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:57.502139 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:57.502118 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:57.502219 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:57.502230 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:57.502248 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:57.502253 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:57.502261 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:57.502318 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:57.502327 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:57.502336 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:57.502342 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:57.561118 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0425 23:04:57.561138 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0425 23:04:57.631061 3778 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0425 23:04:57.645055 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:57.645074 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 23:04:57.645080 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default I0425 23:04:57.746055 3778 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0425 23:04:58.501883 3778 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-h4c87] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-h4c87 for pod on switch ip-10-0-2-186.ec2.internal I0425 23:04:58.501972 3778 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-h4c87 I0425 23:04:58.520183 3778 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-h4c87] pod update took 18.230393ms I0425 23:04:58.520215 3778 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-h4c87] addLogicalPort annotation time took 18.269148ms I0425 23:04:58.521547 3778 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-h4c87] addLogicalPort took 19.684525ms, libovsdb time 919.997µs I0425 23:04:58.610770 3778 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-v9c96] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-5d7684fb5d-v9c96 for pod on switch ip-10-0-2-186.ec2.internal I0425 23:04:58.610889 3778 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-v9c96 I0425 23:04:58.630568 3778 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-v9c96] pod update took 19.705697ms I0425 23:04:58.630596 3778 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-v9c96] addLogicalPort annotation time took 19.740647ms I0425 23:04:58.631778 3778 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-v9c96] addLogicalPort took 21.030621ms, libovsdb time 885.285µs I0425 23:04:58.970085 3778 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-v9c96 4cc6ec43a0f2eea32afa298a892f54f141bbe5971dba4663869afad4f8ce973a network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-v9c96 4cc6ec43a0f2eea32afa298a892f54f141bbe5971dba4663869afad4f8ce973a network default NAD default NAD key ] I0425 23:04:58.980779 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-5d7684fb5d-v9c96, hostIfaceName: 4cc6ec43a0f2eea, network: default, NAD default, SandboxID: "4cc6ec43a0f2eea32afa298a892f54f141bbe5971dba4663869afad4f8ce973a", PCI device ID: , UID: "7805a8c3-3e82-4e58-aa11-ee6de03451a8", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0425 23:04:59.034708 3778 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-v9c96 4cc6ec43a0f2eea32afa298a892f54f141bbe5971dba4663869afad4f8ce973a network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-v9c96 4cc6ec43a0f2eea32afa298a892f54f141bbe5971dba4663869afad4f8ce973a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:71:44:e8:98:cd\",\"name\":\"4cc6ec43a0f2eea\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/908fc95e-88ee-407c-b77f-bdef51e8b8f4\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:04:59.459170 3778 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-h4c87 096da1de78ff2e3f6ac6beaa3096230c6e5036461f00d653cecd1489525db359 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-h4c87 096da1de78ff2e3f6ac6beaa3096230c6e5036461f00d653cecd1489525db359 network default NAD default NAD key ] I0425 23:04:59.470908 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-h4c87, hostIfaceName: 096da1de78ff2e3, network: default, NAD default, SandboxID: "096da1de78ff2e3f6ac6beaa3096230c6e5036461f00d653cecd1489525db359", PCI device ID: , UID: "262a87e5-532a-4963-a260-c9cd277fd4f6", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0425 23:04:59.520492 3778 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-h4c87 096da1de78ff2e3f6ac6beaa3096230c6e5036461f00d653cecd1489525db359 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-h4c87 096da1de78ff2e3f6ac6beaa3096230c6e5036461f00d653cecd1489525db359 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:56:cf:fe:48:5f\",\"name\":\"096da1de78ff2e3\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a771d1a1-102c-4854-b8c5-f011c4ae9231\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:05:01.051623 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0425 23:05:02.542694 3778 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0425 23:05:03.056903 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0425 23:05:03.477729 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0425 23:05:04.358684 3778 base_network_controller_pods.go:487] [default/openshift-console/console-684c94545d-qk57w] creating logical port openshift-console_console-684c94545d-qk57w for pod on switch ip-10-0-2-186.ec2.internal I0425 23:05:04.358782 3778 kube.go:256] Updating pod openshift-console/console-684c94545d-qk57w I0425 23:05:04.376601 3778 pod.go:62] [openshift-console/console-684c94545d-qk57w] pod update took 17.846325ms I0425 23:05:04.376628 3778 base_network_controller_pods.go:951] [default/openshift-console/console-684c94545d-qk57w] addLogicalPort annotation time took 17.876369ms W0425 23:05:04.377172 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0425 23:05:04.377890 3778 pods.go:271] [openshift-console/console-684c94545d-qk57w] addLogicalPort took 19.222112ms, libovsdb time 883.411µs I0425 23:05:04.719852 3778 cni.go:452] [openshift-console/console-684c94545d-qk57w 8627c6a6eb85073e0d50e405158c0d0d4c2ad65afff48391d61f72cfa1623b7b network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-684c94545d-qk57w 8627c6a6eb85073e0d50e405158c0d0d4c2ad65afff48391d61f72cfa1623b7b network default NAD default NAD key ] I0425 23:05:04.736645 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-684c94545d-qk57w, hostIfaceName: 8627c6a6eb85073, network: default, NAD default, SandboxID: "8627c6a6eb85073e0d50e405158c0d0d4c2ad65afff48391d61f72cfa1623b7b", PCI device ID: , UID: "dec586a4-d572-43d1-b6a9-3c8e5727689b", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0425 23:05:04.789631 3778 cni.go:473] [openshift-console/console-684c94545d-qk57w 8627c6a6eb85073e0d50e405158c0d0d4c2ad65afff48391d61f72cfa1623b7b network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-684c94545d-qk57w 8627c6a6eb85073e0d50e405158c0d0d4c2ad65afff48391d61f72cfa1623b7b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:71:c5:97:34:f5\",\"name\":\"8627c6a6eb85073\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/48e90512-2cbb-4055-a768-9117749fa8df\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:05:07.562592 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0425 23:05:10.496726 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0425 23:05:13.514241 3778 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0425 23:05:13.514266 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0425 23:05:13.514274 3778 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0425 23:05:13.552395 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0425 23:05:19.185562 3778 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0425 23:05:19.185584 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0425 23:05:19.185591 3778 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0425 23:05:19.570993 3778 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0425 23:05:19.571016 3778 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0425 23:05:19.571021 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0425 23:05:20.733390 3778 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-56d7454bf5-6fn7t] creating logical port openshift-monitoring_metrics-server-56d7454bf5-6fn7t for pod on switch ip-10-0-2-186.ec2.internal I0425 23:05:20.733483 3778 kube.go:256] Updating pod openshift-monitoring/metrics-server-56d7454bf5-6fn7t I0425 23:05:20.750947 3778 pod.go:62] [openshift-monitoring/metrics-server-56d7454bf5-6fn7t] pod update took 17.487325ms I0425 23:05:20.750973 3778 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-56d7454bf5-6fn7t] addLogicalPort annotation time took 17.520224ms I0425 23:05:20.752088 3778 pods.go:271] [openshift-monitoring/metrics-server-56d7454bf5-6fn7t] addLogicalPort took 18.715752ms, libovsdb time 791.395µs I0425 23:05:20.888739 3778 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-rbtb4] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-rbtb4 for pod on switch ip-10-0-2-186.ec2.internal I0425 23:05:20.888842 3778 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-rbtb4 I0425 23:05:20.909844 3778 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-rbtb4] pod update took 21.033397ms I0425 23:05:20.909871 3778 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-rbtb4] addLogicalPort annotation time took 21.067013ms I0425 23:05:20.911047 3778 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-rbtb4] addLogicalPort took 22.326531ms, libovsdb time 846.67µs I0425 23:05:21.096038 3778 cni.go:452] [openshift-monitoring/metrics-server-56d7454bf5-6fn7t 725d1dd4ffa75afa54b52696867d987ba8e27709d2a62080ed62fc846fb62837 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-56d7454bf5-6fn7t 725d1dd4ffa75afa54b52696867d987ba8e27709d2a62080ed62fc846fb62837 network default NAD default NAD key ] I0425 23:05:21.105898 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-56d7454bf5-6fn7t, hostIfaceName: 725d1dd4ffa75af, network: default, NAD default, SandboxID: "725d1dd4ffa75afa54b52696867d987ba8e27709d2a62080ed62fc846fb62837", PCI device ID: , UID: "057dfe0c-71fa-4ebf-9eee-b2596f46c99f", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0425 23:05:21.156441 3778 cni.go:473] [openshift-monitoring/metrics-server-56d7454bf5-6fn7t 725d1dd4ffa75afa54b52696867d987ba8e27709d2a62080ed62fc846fb62837 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-56d7454bf5-6fn7t 725d1dd4ffa75afa54b52696867d987ba8e27709d2a62080ed62fc846fb62837 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:b6:14:ae:b8:a7\",\"name\":\"725d1dd4ffa75af\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9e0aa3d-225e-4cc9-b6a1-37bd038b9358\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:05:21.190701 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0425 23:05:21.217411 3778 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-684c94545d-bqbp7 IP address from the namespace address-set, err: pod openshift-console/console-684c94545d-bqbp7: no pod IPs found I0425 23:05:21.542357 3778 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-rbtb4 f51a028175bcc62f905df68137398c0a19b6bf879a8ca7212a18e98c16556a7c network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-rbtb4 f51a028175bcc62f905df68137398c0a19b6bf879a8ca7212a18e98c16556a7c network default NAD default NAD key ] I0425 23:05:21.552656 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-rbtb4, hostIfaceName: f51a028175bcc62, network: default, NAD default, SandboxID: "f51a028175bcc62f905df68137398c0a19b6bf879a8ca7212a18e98c16556a7c", PCI device ID: , UID: "9c555a7d-1240-4d45-ac7c-3955eae00907", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0425 23:05:21.603131 3778 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-rbtb4 f51a028175bcc62f905df68137398c0a19b6bf879a8ca7212a18e98c16556a7c network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-rbtb4 f51a028175bcc62f905df68137398c0a19b6bf879a8ca7212a18e98c16556a7c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:7e:f0:3c:ce:04\",\"name\":\"f51a028175bcc62\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2ee1b08e-b387-477c-959e-c7069bd95b99\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:05:22.319109 3778 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-186.ec2.internal I0425 23:05:22.319296 3778 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0425 23:05:22.343409 3778 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 24.15324ms I0425 23:05:22.343442 3778 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 24.192024ms I0425 23:05:22.344937 3778 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 25.865027ms, libovsdb time 1.049702ms I0425 23:05:22.970979 3778 cni.go:452] [openshift-monitoring/prometheus-k8s-1 e7e698dc406cec7a336eab578a71fcfe432c9a5b4ec73cff113e6fe8901f687c network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 e7e698dc406cec7a336eab578a71fcfe432c9a5b4ec73cff113e6fe8901f687c network default NAD default NAD key ] I0425 23:05:22.981174 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: e7e698dc406cec7, network: default, NAD default, SandboxID: "e7e698dc406cec7a336eab578a71fcfe432c9a5b4ec73cff113e6fe8901f687c", PCI device ID: , UID: "d0c02566-9b2e-4df6-8e14-5ebbf39986dc", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0425 23:05:23.035171 3778 cni.go:473] [openshift-monitoring/prometheus-k8s-1 e7e698dc406cec7a336eab578a71fcfe432c9a5b4ec73cff113e6fe8901f687c network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 e7e698dc406cec7a336eab578a71fcfe432c9a5b4ec73cff113e6fe8901f687c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:67:80:a0:22:ac\",\"name\":\"e7e698dc406cec7\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9f37fc9f-3050-4f20-beaa-c81f6201312b\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:05:23.602434 3778 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:05:23.602456 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:05:23.627113 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0425 23:05:23.666043 3778 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 23:05:23.666061 3778 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 23:05:23.666066 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 23:05:23.666070 3778 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 23:05:24.166850 3778 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 23:05:24.166870 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 23:05:24.166875 3778 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 23:05:24.166879 3778 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 23:05:24.246933 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0425 23:05:24.646443 3778 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:05:24.646461 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:05:26.451991 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0425 23:05:26.479623 3778 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-689c8bc4db-g289n IP address from the namespace address-set, err: pod openshift-console/console-689c8bc4db-g289n: no pod IPs found I0425 23:05:27.263071 3778 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0425 23:05:27.501444 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0425 23:05:27.501472 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0425 23:05:27.571259 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0425 23:05:29.184886 3778 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6b57c9b684-vtnnk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:05:29.185501 3778 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6b57c9b684-vtnnk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:05:29.197718 3778 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6b57c9b684-vtnnk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:05:29.197739 3778 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6b57c9b684-vtnnk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:05:29.203617 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6b57c9b684-vtnnk I0425 23:05:29.203645 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6b57c9b684-vtnnk I0425 23:05:30.369156 3778 obj_retry.go:498] Detected object openshift-console/console-5f967f89dd-t2v9h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:05:30.369716 3778 obj_retry.go:498] Detected object openshift-console/console-5f967f89dd-t2v9h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0425 23:05:30.376555 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0425 23:05:31.195349 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0425 23:05:31.206442 3778 obj_retry.go:492] Detected object openshift-console/console-5f967f89dd-t2v9h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:05:31.206465 3778 obj_retry.go:492] Detected object openshift-console/console-5f967f89dd-t2v9h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:05:31.212275 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5f967f89dd-t2v9h I0425 23:05:31.212295 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5f967f89dd-t2v9h W0425 23:05:38.225623 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0425 23:05:41.246556 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0425 23:05:41.246579 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0425 23:05:42.681340 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0425 23:05:44.637921 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0425 23:05:46.504967 3778 obj_retry.go:498] Detected object openshift-console/console-5f967f89dd-c9bhm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:05:46.505561 3778 obj_retry.go:498] Detected object openshift-console/console-5f967f89dd-c9bhm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:05:46.692040 3778 obj_retry.go:492] Detected object openshift-console/console-5f967f89dd-c9bhm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:05:46.692063 3778 obj_retry.go:492] Detected object openshift-console/console-5f967f89dd-c9bhm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:05:46.697578 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5f967f89dd-c9bhm I0425 23:05:46.697601 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5f967f89dd-c9bhm I0425 23:05:51.613683 3778 cni.go:452] [openshift-console/console-684c94545d-qk57w 8627c6a6eb85073e0d50e405158c0d0d4c2ad65afff48391d61f72cfa1623b7b network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-684c94545d-qk57w 8627c6a6eb85073e0d50e405158c0d0d4c2ad65afff48391d61f72cfa1623b7b network default NAD default NAD key ] I0425 23:05:51.694916 3778 cni.go:473] [openshift-console/console-684c94545d-qk57w 8627c6a6eb85073e0d50e405158c0d0d4c2ad65afff48391d61f72cfa1623b7b network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-684c94545d-qk57w 8627c6a6eb85073e0d50e405158c0d0d4c2ad65afff48391d61f72cfa1623b7b network default NAD default NAD key default], result "{}", err I0425 23:05:51.752493 3778 obj_retry.go:498] Detected object openshift-console/console-684c94545d-qk57w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:05:51.752523 3778 pods.go:176] Deleting pod: openshift-console/console-684c94545d-qk57w I0425 23:05:51.753721 3778 pods.go:236] Attempting to release IPs for pod: openshift-console/console-684c94545d-qk57w, ips: 10.128.0.16 I0425 23:05:51.753746 3778 obj_retry.go:498] Detected object openshift-console/console-684c94545d-qk57w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:05:51.762578 3778 base_network_controller_pods.go:487] [default/openshift-console/console-6c74477659-q4jkd] creating logical port openshift-console_console-6c74477659-q4jkd for pod on switch ip-10-0-2-186.ec2.internal I0425 23:05:51.762744 3778 kube.go:256] Updating pod openshift-console/console-6c74477659-q4jkd I0425 23:05:51.777426 3778 pod.go:62] [openshift-console/console-6c74477659-q4jkd] pod update took 14.727629ms I0425 23:05:51.777451 3778 base_network_controller_pods.go:951] [default/openshift-console/console-6c74477659-q4jkd] addLogicalPort annotation time took 14.760511ms I0425 23:05:51.778607 3778 pods.go:271] [openshift-console/console-6c74477659-q4jkd] addLogicalPort took 16.047571ms, libovsdb time 842.429µs I0425 23:05:52.053171 3778 obj_retry.go:492] Detected object openshift-console/console-684c94545d-qk57w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:05:52.053203 3778 obj_retry.go:492] Detected object openshift-console/console-684c94545d-qk57w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:05:52.061008 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-684c94545d-qk57w I0425 23:05:52.061036 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-684c94545d-qk57w I0425 23:05:52.122543 3778 cni.go:452] [openshift-console/console-6c74477659-q4jkd b421289d2867349856463618b396635945c3a17ba9e4ca3e410edd4c335103d5 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6c74477659-q4jkd b421289d2867349856463618b396635945c3a17ba9e4ca3e410edd4c335103d5 network default NAD default NAD key ] I0425 23:05:52.136643 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6c74477659-q4jkd, hostIfaceName: b421289d2867349, network: default, NAD default, SandboxID: "b421289d2867349856463618b396635945c3a17ba9e4ca3e410edd4c335103d5", PCI device ID: , UID: "04ccb385-e284-4317-8a44-7998565ddd09", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0425 23:05:52.190322 3778 cni.go:473] [openshift-console/console-6c74477659-q4jkd b421289d2867349856463618b396635945c3a17ba9e4ca3e410edd4c335103d5 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6c74477659-q4jkd b421289d2867349856463618b396635945c3a17ba9e4ca3e410edd4c335103d5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:7c:97:1c:0e:69\",\"name\":\"b421289d2867349\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f995b3a9-4d48-49bb-9379-0ac5722ee884\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:05:52.730329 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0425 23:05:57.759149 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0425 23:05:57.759175 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0425 23:05:57.795192 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0425 23:05:57.795230 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0425 23:05:58.102193 3778 cni.go:452] [openshift-multus/network-metrics-daemon-6tvfc b26af808f270cdb28a08453cbb8fdf04f0592c570ca9a09c27dccca58f32d858 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-6tvfc b26af808f270cdb28a08453cbb8fdf04f0592c570ca9a09c27dccca58f32d858 network default NAD default NAD key ] I0425 23:05:58.111761 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-6tvfc, hostIfaceName: b26af808f270cdb, network: default, NAD default, SandboxID: "b26af808f270cdb28a08453cbb8fdf04f0592c570ca9a09c27dccca58f32d858", PCI device ID: , UID: "35129562-2f55-46d5-a60e-59e4dfb7741b", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0425 23:05:58.163320 3778 cni.go:473] [openshift-multus/network-metrics-daemon-6tvfc b26af808f270cdb28a08453cbb8fdf04f0592c570ca9a09c27dccca58f32d858 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-6tvfc b26af808f270cdb28a08453cbb8fdf04f0592c570ca9a09c27dccca58f32d858 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:4b:11:3e:2a:bd\",\"name\":\"b26af808f270cdb\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/86cc6eda-a3e4-4e96-ae01-68936666565d\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:06:02.808849 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0425 23:06:13.256158 3778 cni.go:452] [openshift-ingress/router-default-8668968b4f-x7pdx fb84f99e8dcb2a3f0a0f21061715ba7bbf016f83acc464fad15191fbe4a851fc network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-8668968b4f-x7pdx fb84f99e8dcb2a3f0a0f21061715ba7bbf016f83acc464fad15191fbe4a851fc network default NAD default NAD key ] I0425 23:06:13.332602 3778 cni.go:473] [openshift-ingress/router-default-8668968b4f-x7pdx fb84f99e8dcb2a3f0a0f21061715ba7bbf016f83acc464fad15191fbe4a851fc network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-8668968b4f-x7pdx fb84f99e8dcb2a3f0a0f21061715ba7bbf016f83acc464fad15191fbe4a851fc network default NAD default NAD key default], result "{}", err I0425 23:06:13.380153 3778 obj_retry.go:498] Detected object openshift-ingress/router-default-8668968b4f-x7pdx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:06:13.380202 3778 pods.go:176] Deleting pod: openshift-ingress/router-default-8668968b4f-x7pdx I0425 23:06:13.381539 3778 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-8668968b4f-x7pdx, ips: 10.128.0.10 I0425 23:06:13.381567 3778 obj_retry.go:498] Detected object openshift-ingress/router-default-8668968b4f-x7pdx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:06:13.834205 3778 obj_retry.go:492] Detected object openshift-ingress/router-default-8668968b4f-x7pdx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:06:13.834229 3778 obj_retry.go:492] Detected object openshift-ingress/router-default-8668968b4f-x7pdx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:06:13.839894 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-8668968b4f-x7pdx I0425 23:06:13.839927 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-8668968b4f-x7pdx W0425 23:06:22.885585 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:06:22.885610 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:06:22.885625 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:06:22.885631 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:06:22.914300 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:06:22.914321 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0425 23:06:23.072645 3778 obj_retry.go:498] Detected object openshift-console/console-689c8bc4db-gxch2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:06:23.073189 3778 obj_retry.go:498] Detected object openshift-console/console-689c8bc4db-gxch2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:06:23.393843 3778 obj_retry.go:492] Detected object openshift-console/console-689c8bc4db-gxch2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:06:23.393864 3778 obj_retry.go:492] Detected object openshift-console/console-689c8bc4db-gxch2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:06:23.401327 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-689c8bc4db-gxch2 I0425 23:06:23.401352 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-689c8bc4db-gxch2 I0425 23:06:29.509346 3778 cni.go:452] [openshift-dns/dns-default-vzr5r e1c7ac7b7bc30806179f0a206e04989da7603220ec70b56d99cab7b98109a95a network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-vzr5r e1c7ac7b7bc30806179f0a206e04989da7603220ec70b56d99cab7b98109a95a network default NAD default NAD key ] I0425 23:06:29.518742 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-vzr5r, hostIfaceName: e1c7ac7b7bc3080, network: default, NAD default, SandboxID: "e1c7ac7b7bc30806179f0a206e04989da7603220ec70b56d99cab7b98109a95a", PCI device ID: , UID: "9ec59f18-5fbf-452b-a263-79619c43a2a3", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0425 23:06:29.568551 3778 cni.go:473] [openshift-dns/dns-default-vzr5r e1c7ac7b7bc30806179f0a206e04989da7603220ec70b56d99cab7b98109a95a network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-vzr5r e1c7ac7b7bc30806179f0a206e04989da7603220ec70b56d99cab7b98109a95a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:49:b5:bc:3b:25\",\"name\":\"e1c7ac7b7bc3080\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0fac07ba-f717-4058-9de8-b50c7e7d6533\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:06:37.759262 3778 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:06:37.759795 3778 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:06:38.470114 3778 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:06:38.470141 3778 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:06:38.475429 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0425 23:06:38.475449 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 W0425 23:06:41.443536 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0425 23:06:41.443561 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0425 23:06:41.443569 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0425 23:06:41.443581 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0425 23:06:41.443586 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0425 23:06:41.443590 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0425 23:06:41.870246 3778 cni.go:452] [openshift-monitoring/prometheus-k8s-1 e7e698dc406cec7a336eab578a71fcfe432c9a5b4ec73cff113e6fe8901f687c network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 e7e698dc406cec7a336eab578a71fcfe432c9a5b4ec73cff113e6fe8901f687c network default NAD default NAD key ] W0425 23:06:41.919256 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0425 23:06:41.919276 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0425 23:06:41.919281 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0425 23:06:41.949784 3778 cni.go:473] [openshift-monitoring/prometheus-k8s-1 e7e698dc406cec7a336eab578a71fcfe432c9a5b4ec73cff113e6fe8901f687c network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 e7e698dc406cec7a336eab578a71fcfe432c9a5b4ec73cff113e6fe8901f687c network default NAD default NAD key default], result "{}", err I0425 23:06:42.007665 3778 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:06:42.007708 3778 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0425 23:06:42.008944 3778 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.128.0.19 I0425 23:06:42.008978 3778 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:06:42.959891 3778 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:06:42.959921 3778 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:06:42.965638 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0425 23:06:42.965665 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0425 23:06:43.004614 3778 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-186.ec2.internal I0425 23:06:43.004792 3778 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0425 23:06:43.027198 3778 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 22.462159ms I0425 23:06:43.027228 3778 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 22.497919ms I0425 23:06:43.028631 3778 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 24.03395ms, libovsdb time 999.189µs I0425 23:06:43.366729 3778 cni.go:452] [openshift-monitoring/prometheus-k8s-1 a2bd7700524f1c6427883bd7ae15ca4a8c1f8f3f5ec436b4be766b37de84bd12 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 a2bd7700524f1c6427883bd7ae15ca4a8c1f8f3f5ec436b4be766b37de84bd12 network default NAD default NAD key ] I0425 23:06:43.376540 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: a2bd7700524f1c6, network: default, NAD default, SandboxID: "a2bd7700524f1c6427883bd7ae15ca4a8c1f8f3f5ec436b4be766b37de84bd12", PCI device ID: , UID: "89c6db17-96b3-4eb1-ba85-2ddbfda7ce41", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0425 23:06:43.428997 3778 cni.go:473] [openshift-monitoring/prometheus-k8s-1 a2bd7700524f1c6427883bd7ae15ca4a8c1f8f3f5ec436b4be766b37de84bd12 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 a2bd7700524f1c6427883bd7ae15ca4a8c1f8f3f5ec436b4be766b37de84bd12 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:61:30:67:6c:ad\",\"name\":\"a2bd7700524f1c6\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc01a446-628b-4ba0-b84f-6ff221827a89\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:07:10.228122 3778 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:07:10.228710 3778 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:07:11.054181 3778 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:07:11.054227 3778 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:07:11.061373 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0425 23:07:11.061396 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0425 23:07:11.164055 3778 cni.go:452] [openshift-console/console-6c74477659-q4jkd b421289d2867349856463618b396635945c3a17ba9e4ca3e410edd4c335103d5 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-6c74477659-q4jkd b421289d2867349856463618b396635945c3a17ba9e4ca3e410edd4c335103d5 network default NAD default NAD key ] I0425 23:07:11.242705 3778 cni.go:473] [openshift-console/console-6c74477659-q4jkd b421289d2867349856463618b396635945c3a17ba9e4ca3e410edd4c335103d5 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-6c74477659-q4jkd b421289d2867349856463618b396635945c3a17ba9e4ca3e410edd4c335103d5 network default NAD default NAD key default], result "{}", err I0425 23:07:11.287701 3778 obj_retry.go:498] Detected object openshift-console/console-6c74477659-q4jkd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:07:11.287730 3778 pods.go:176] Deleting pod: openshift-console/console-6c74477659-q4jkd I0425 23:07:11.289084 3778 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6c74477659-q4jkd, ips: 10.128.0.20 I0425 23:07:11.289127 3778 obj_retry.go:498] Detected object openshift-console/console-6c74477659-q4jkd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:07:11.303845 3778 base_network_controller_pods.go:487] [default/openshift-console/console-7745fbf5d4-psnbf] creating logical port openshift-console_console-7745fbf5d4-psnbf for pod on switch ip-10-0-2-186.ec2.internal I0425 23:07:11.303971 3778 kube.go:256] Updating pod openshift-console/console-7745fbf5d4-psnbf I0425 23:07:11.319267 3778 pod.go:62] [openshift-console/console-7745fbf5d4-psnbf] pod update took 15.319378ms I0425 23:07:11.319299 3778 base_network_controller_pods.go:951] [default/openshift-console/console-7745fbf5d4-psnbf] addLogicalPort annotation time took 15.353527ms I0425 23:07:11.320953 3778 pods.go:271] [openshift-console/console-7745fbf5d4-psnbf] addLogicalPort took 17.121083ms, libovsdb time 1.26055ms I0425 23:07:11.663217 3778 cni.go:452] [openshift-console/console-7745fbf5d4-psnbf 93d57a651307fbf624a4705532662c689d6fb7c2151a5660922687d38ccdf7e6 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7745fbf5d4-psnbf 93d57a651307fbf624a4705532662c689d6fb7c2151a5660922687d38ccdf7e6 network default NAD default NAD key ] I0425 23:07:11.672857 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7745fbf5d4-psnbf, hostIfaceName: 93d57a651307fbf, network: default, NAD default, SandboxID: "93d57a651307fbf624a4705532662c689d6fb7c2151a5660922687d38ccdf7e6", PCI device ID: , UID: "cc5b32e4-e550-494a-b95c-9934ad3ea074", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0425 23:07:11.721935 3778 cni.go:473] [openshift-console/console-7745fbf5d4-psnbf 93d57a651307fbf624a4705532662c689d6fb7c2151a5660922687d38ccdf7e6 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7745fbf5d4-psnbf 93d57a651307fbf624a4705532662c689d6fb7c2151a5660922687d38ccdf7e6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d2:a0:bb:b6:86:86\",\"name\":\"93d57a651307fbf\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0ee22540-765c-4a55-9bb2-a2b7cc880715\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:07:12.087449 3778 obj_retry.go:492] Detected object openshift-console/console-6c74477659-q4jkd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:07:12.087481 3778 obj_retry.go:492] Detected object openshift-console/console-6c74477659-q4jkd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:07:12.097585 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6c74477659-q4jkd I0425 23:07:12.097611 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6c74477659-q4jkd I0425 23:07:21.867416 3778 obj_retry.go:498] Detected object openshift-console/console-6c74477659-jnnx9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:07:21.867948 3778 obj_retry.go:498] Detected object openshift-console/console-6c74477659-jnnx9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:07:22.103535 3778 obj_retry.go:492] Detected object openshift-console/console-6c74477659-jnnx9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:07:22.103561 3778 obj_retry.go:492] Detected object openshift-console/console-6c74477659-jnnx9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:07:22.111340 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6c74477659-jnnx9 I0425 23:07:22.111363 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6c74477659-jnnx9 W0425 23:07:22.118171 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0425 23:07:25.718194 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 W0425 23:07:44.224339 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 23:07:44.224365 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0425 23:07:45.529533 3778 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:07:45.530116 3778 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:07:46.208384 3778 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:07:46.208411 3778 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:07:46.215602 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0425 23:07:46.215636 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0425 23:07:57.707819 3778 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=5 I0425 23:08:03.620798 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=136 I0425 23:08:11.292748 3778 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc 74eccb0863bc8d346f697ad679eb7db5a37a96699b16dce1a8f3592fc021e34b network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc 74eccb0863bc8d346f697ad679eb7db5a37a96699b16dce1a8f3592fc021e34b network default NAD default NAD key ] I0425 23:08:11.372965 3778 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc 74eccb0863bc8d346f697ad679eb7db5a37a96699b16dce1a8f3592fc021e34b network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc 74eccb0863bc8d346f697ad679eb7db5a37a96699b16dce1a8f3592fc021e34b network default NAD default NAD key default], result "{}", err I0425 23:08:11.415065 3778 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:08:11.415112 3778 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc I0425 23:08:11.416377 3778 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc, ips: 10.128.0.8 I0425 23:08:11.416413 3778 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:08:12.303013 3778 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:08:12.303035 3778 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619300-zwscc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:08:22.744998 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0425 23:08:26.707442 3778 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0425 23:08:28.279086 3778 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0425 23:08:50.723761 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0425 23:08:51.744813 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0425 23:09:08.728227 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0425 23:09:15.711667 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0425 23:09:56.739787 3778 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0425 23:10:00.187352 3778 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29619310-btdcj for pod on switch ip-10-0-2-186.ec2.internal I0425 23:10:00.187464 3778 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj I0425 23:10:00.213997 3778 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj] pod update took 26.558594ms I0425 23:10:00.214022 3778 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj] addLogicalPort annotation time took 26.586671ms I0425 23:10:00.215376 3778 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj] addLogicalPort took 28.038173ms, libovsdb time 919.445µs I0425 23:10:00.552789 3778 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj dc1978427e9cd30438fed15fcd731cf5ded9b660e5e7c5b68234215969e86098 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj dc1978427e9cd30438fed15fcd731cf5ded9b660e5e7c5b68234215969e86098 network default NAD default NAD key ] I0425 23:10:00.562024 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29619310-btdcj, hostIfaceName: dc1978427e9cd30, network: default, NAD default, SandboxID: "dc1978427e9cd30438fed15fcd731cf5ded9b660e5e7c5b68234215969e86098", PCI device ID: , UID: "4a4ee8d7-6f5a-405d-a2f0-91cddead699c", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0425 23:10:00.610973 3778 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj dc1978427e9cd30438fed15fcd731cf5ded9b660e5e7c5b68234215969e86098 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj dc1978427e9cd30438fed15fcd731cf5ded9b660e5e7c5b68234215969e86098 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:60:ac:78:6d:bd\",\"name\":\"dc1978427e9cd30\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4a855fb-e1a6-4e57-8a6d-0763f29f5df2\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:10:03.702724 3778 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj dc1978427e9cd30438fed15fcd731cf5ded9b660e5e7c5b68234215969e86098 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj dc1978427e9cd30438fed15fcd731cf5ded9b660e5e7c5b68234215969e86098 network default NAD default NAD key ] I0425 23:10:03.777235 3778 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj dc1978427e9cd30438fed15fcd731cf5ded9b660e5e7c5b68234215969e86098 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj dc1978427e9cd30438fed15fcd731cf5ded9b660e5e7c5b68234215969e86098 network default NAD default NAD key default], result "{}", err I0425 23:10:03.817237 3778 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:10:03.817280 3778 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj I0425 23:10:03.818588 3778 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj, ips: 10.128.0.23 I0425 23:10:03.818619 3778 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:10:04.716125 3778 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:10:04.716148 3778 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619310-btdcj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:10:33.624979 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=84 I0425 23:10:35.625499 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0425 23:10:53.628533 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=16 I0425 23:11:34.624963 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=74 I0425 23:11:48.734532 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0425 23:12:10.630609 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=665 I0425 23:13:50.710992 3778 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0425 23:14:15.711269 3778 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0425 23:14:37.731575 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0425 23:14:55.721945 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0425 23:15:19.743480 3778 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0425 23:15:54.748484 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0425 23:16:02.632593 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=7 I0425 23:16:12.627628 3778 namespace.go:98] [tsf] adding namespace I0425 23:16:12.628664 3778 namespace.go:102] [tsf] adding namespace took 1.013447ms I0425 23:16:12.641511 3778 namespace.go:142] [tsf] updating namespace I0425 23:16:12.653378 3778 namespace.go:142] [tsf] updating namespace I0425 23:16:16.375708 3778 namespace.go:98] [tsf-tpa] adding namespace I0425 23:16:16.375770 3778 namespace.go:98] [tsf-keycloak] adding namespace I0425 23:16:16.375892 3778 namespace.go:98] [konflux-operator] adding namespace I0425 23:16:16.375893 3778 namespace.go:98] [rhtpa-operator] adding namespace I0425 23:16:16.376081 3778 namespace.go:98] [rhbk-operator] adding namespace I0425 23:16:16.376267 3778 namespace.go:98] [cert-manager-operator] adding namespace I0425 23:16:16.376762 3778 namespace.go:102] [tsf-tpa] adding namespace took 1.028451ms I0425 23:16:16.377356 3778 namespace.go:102] [tsf-keycloak] adding namespace took 1.57227ms I0425 23:16:16.377846 3778 namespace.go:98] [tsf-tas] adding namespace I0425 23:16:16.377946 3778 namespace.go:102] [konflux-operator] adding namespace took 2.037028ms I0425 23:16:16.378561 3778 namespace.go:102] [rhtpa-operator] adding namespace took 2.648393ms I0425 23:16:16.379106 3778 namespace.go:102] [rhbk-operator] adding namespace took 2.977309ms I0425 23:16:16.379135 3778 namespace.go:98] [konflux-ui] adding namespace I0425 23:16:16.379666 3778 namespace.go:102] [konflux-ui] adding namespace took 526.313µs I0425 23:16:16.380195 3778 namespace.go:102] [cert-manager-operator] adding namespace took 3.911949ms I0425 23:16:16.380620 3778 namespace.go:102] [tsf-tas] adding namespace took 2.759653ms I0425 23:16:16.386481 3778 namespace.go:142] [tsf-tpa] updating namespace I0425 23:16:16.398442 3778 namespace.go:142] [tsf-tpa] updating namespace I0425 23:16:16.416579 3778 namespace.go:142] [tsf-keycloak] updating namespace I0425 23:16:16.423440 3778 namespace.go:142] [tsf-keycloak] updating namespace I0425 23:16:16.430051 3778 namespace.go:142] [konflux-operator] updating namespace I0425 23:16:16.431682 3778 namespace.go:142] [rhbk-operator] updating namespace I0425 23:16:16.438296 3778 namespace.go:142] [rhtpa-operator] updating namespace I0425 23:16:16.442616 3778 namespace.go:142] [konflux-operator] updating namespace I0425 23:16:16.442712 3778 namespace.go:142] [konflux-ui] updating namespace I0425 23:16:16.444608 3778 namespace.go:142] [cert-manager-operator] updating namespace I0425 23:16:16.448024 3778 namespace.go:142] [tsf-tas] updating namespace I0425 23:16:16.462079 3778 namespace.go:142] [tsf-keycloak] updating namespace I0425 23:16:16.462232 3778 namespace.go:142] [konflux-operator] updating namespace I0425 23:16:16.491582 3778 namespace.go:142] [rhbk-operator] updating namespace I0425 23:16:16.494801 3778 namespace.go:142] [rhbk-operator] updating namespace I0425 23:16:16.504555 3778 namespace.go:142] [konflux-ui] updating namespace I0425 23:16:16.515587 3778 namespace.go:142] [konflux-ui] updating namespace I0425 23:16:16.521611 3778 namespace.go:142] [cert-manager-operator] updating namespace I0425 23:16:16.534680 3778 namespace.go:142] [cert-manager-operator] updating namespace I0425 23:16:16.534736 3778 namespace.go:142] [tsf-tas] updating namespace I0425 23:16:16.546492 3778 namespace.go:142] [tsf-tas] updating namespace I0425 23:16:16.553593 3778 namespace.go:142] [rhtpa-operator] updating namespace I0425 23:16:16.559674 3778 namespace.go:142] [rhtpa-operator] updating namespace I0425 23:16:16.579579 3778 namespace.go:142] [tsf-tpa] updating namespace I0425 23:16:17.460540 3778 namespace.go:142] [tsf-tpa] updating namespace I0425 23:16:17.470910 3778 namespace.go:142] [tsf-keycloak] updating namespace I0425 23:16:19.748981 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0425 23:16:23.528556 3778 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51sf9m9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:23.529066 3778 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51sf9m9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:23.529118 3778 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51sf9m9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:23.529131 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 23:16:23.529144 3778 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51sf9m9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:23.529150 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 23:16:23.556777 3778 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158dnmv2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:23.557299 3778 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158dnmv2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:23.557327 3778 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158dnmv2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:23.557335 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 23:16:23.557345 3778 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158dnmv2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:23.557349 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 23:16:24.553143 3778 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51sf9m9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:24.553168 3778 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51sf9m9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:24.553183 3778 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51sf9m9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:24.553195 3778 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51sf9m9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:24.580443 3778 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158dnmv2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:24.580454 3778 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158dnmv2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:24.580471 3778 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158dnmv2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:24.580477 3778 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158dnmv2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:26.984607 3778 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2jxwcf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:26.985129 3778 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2jxwcf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:26.985154 3778 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2jxwcf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:26.985162 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 23:16:26.985173 3778 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2jxwcf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:26.985177 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 23:16:28.009166 3778 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2jxwcf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:28.009196 3778 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2jxwcf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:28.009211 3778 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2jxwcf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:28.009222 3778 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2jxwcf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:29.598616 3778 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgltq8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.599207 3778 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgltq8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.599235 3778 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgltq8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.599243 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 23:16:29.599255 3778 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgltq8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.599262 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 23:16:29.634936 3778 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dcnq2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.635455 3778 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dcnq2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.635476 3778 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dcnq2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.635482 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 23:16:29.635490 3778 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dcnq2 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.635494 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 23:16:29.662518 3778 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136cgfmp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.662998 3778 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136cgfmp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.663019 3778 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136cgfmp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.663024 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 23:16:29.663030 3778 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136cgfmp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 23:16:29.663034 3778 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 23:16:30.624975 3778 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgltq8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.624997 3778 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgltq8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.625011 3778 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgltq8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.625023 3778 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgltq8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.663893 3778 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dcnq2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.663913 3778 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dcnq2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.663925 3778 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dcnq2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.663936 3778 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dcnq2 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.696074 3778 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136cgfmp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.696113 3778 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136cgfmp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.696126 3778 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136cgfmp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:30.696139 3778 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136cgfmp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:16:31.628909 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0425 23:16:32.978741 3778 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-q5w2q] creating logical port tsf-tpa_rhtpa-operator-controller-manager-586cbb496b-q5w2q for pod on switch ip-10-0-2-186.ec2.internal I0425 23:16:32.978855 3778 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-q5w2q I0425 23:16:32.999792 3778 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-q5w2q] pod update took 20.961828ms I0425 23:16:32.999818 3778 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-q5w2q] addLogicalPort annotation time took 20.994106ms I0425 23:16:33.001018 3778 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-q5w2q] addLogicalPort took 22.296417ms, libovsdb time 864.321µs I0425 23:16:33.364861 3778 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-q5w2q 7bc70b888998fbcaa8ec0ef96e74a9a4f926d50f8feffdd8583cfe248ca55e34 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-q5w2q 7bc70b888998fbcaa8ec0ef96e74a9a4f926d50f8feffdd8583cfe248ca55e34 network default NAD default NAD key ] I0425 23:16:33.375087 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-586cbb496b-q5w2q, hostIfaceName: 7bc70b888998fbc, network: default, NAD default, SandboxID: "7bc70b888998fbcaa8ec0ef96e74a9a4f926d50f8feffdd8583cfe248ca55e34", PCI device ID: , UID: "4556ac69-9ee1-4e64-8540-f10dc19e9463", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0425 23:16:33.427525 3778 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-q5w2q 7bc70b888998fbcaa8ec0ef96e74a9a4f926d50f8feffdd8583cfe248ca55e34 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-q5w2q 7bc70b888998fbcaa8ec0ef96e74a9a4f926d50f8feffdd8583cfe248ca55e34 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:d5:f3:94:a9:da\",\"name\":\"7bc70b888998fbc\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f0e5f1e-8404-44eb-8079-54399e89c881\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:16:36.204785 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0425 23:16:36.476847 3778 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-4f97h] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-4f97h for pod on switch ip-10-0-2-186.ec2.internal I0425 23:16:36.476979 3778 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-4f97h I0425 23:16:36.536602 3778 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-4f97h] pod update took 59.655304ms I0425 23:16:36.536632 3778 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-4f97h] addLogicalPort annotation time took 59.691008ms I0425 23:16:36.537815 3778 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-4f97h] addLogicalPort took 60.985224ms, libovsdb time 856.592µs I0425 23:16:36.838788 3778 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-4f97h c4fb8685c3e1d2ef8415620a278f3ab016eebd34a19c8b49535cfb63b72f4202 network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-4f97h c4fb8685c3e1d2ef8415620a278f3ab016eebd34a19c8b49535cfb63b72f4202 network default NAD default NAD key ] I0425 23:16:36.849962 3778 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-4f97h, hostIfaceName: c4fb8685c3e1d2e, network: default, NAD default, SandboxID: "c4fb8685c3e1d2ef8415620a278f3ab016eebd34a19c8b49535cfb63b72f4202", PCI device ID: , UID: "18da441d-ba54-4653-a45d-c97de3b7d970", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0425 23:16:36.906204 3778 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-4f97h c4fb8685c3e1d2ef8415620a278f3ab016eebd34a19c8b49535cfb63b72f4202 network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-4f97h c4fb8685c3e1d2ef8415620a278f3ab016eebd34a19c8b49535cfb63b72f4202 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:be:c0:52:6e:23\",\"name\":\"c4fb8685c3e1d2e\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/371adbee-01fe-404f-a809-7097e97bbd33\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:16:41.242976 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0425 23:16:41.262535 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0425 23:16:42.549588 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0425 23:16:46.001825 3778 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-s75cq] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-s75cq for pod on switch ip-10-0-2-186.ec2.internal I0425 23:16:46.001950 3778 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-s75cq I0425 23:16:46.032019 3778 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-s75cq] pod update took 30.096777ms I0425 23:16:46.032046 3778 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-s75cq] addLogicalPort annotation time took 30.129033ms I0425 23:16:46.033935 3778 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-s75cq] addLogicalPort took 32.123152ms, libovsdb time 1.198721ms I0425 23:16:46.358260 3778 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-s75cq c34e02139ddb10900b16ce8f3ea9eac6c6a03a8500c884d3fecb51f64c50a635 network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-s75cq c34e02139ddb10900b16ce8f3ea9eac6c6a03a8500c884d3fecb51f64c50a635 network default NAD default NAD key ] I0425 23:16:46.368069 3778 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-s75cq, hostIfaceName: c34e02139ddb109, network: default, NAD default, SandboxID: "c34e02139ddb10900b16ce8f3ea9eac6c6a03a8500c884d3fecb51f64c50a635", PCI device ID: , UID: "5c125a9d-9c2a-4b31-ad27-2f728a7083dc", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0425 23:16:46.421749 3778 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-s75cq c34e02139ddb10900b16ce8f3ea9eac6c6a03a8500c884d3fecb51f64c50a635 network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-s75cq c34e02139ddb10900b16ce8f3ea9eac6c6a03a8500c884d3fecb51f64c50a635 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:5d:c4:a6:fb:c5\",\"name\":\"c34e02139ddb109\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3dea6ce1-ab81-4035-bfb1-a03378873637\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:16:47.150703 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0425 23:16:47.256731 3778 namespace.go:142] [openshift-operators] updating namespace I0425 23:16:47.260857 3778 namespace.go:142] [openshift-operators] updating namespace I0425 23:16:48.579607 3778 namespace.go:98] [cert-manager] adding namespace I0425 23:16:48.580561 3778 namespace.go:102] [cert-manager] adding namespace took 921.087µs I0425 23:16:48.602834 3778 namespace.go:142] [cert-manager] updating namespace I0425 23:16:48.618248 3778 namespace.go:142] [cert-manager] updating namespace W0425 23:16:50.236463 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0425 23:16:50.263547 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0425 23:16:50.280557 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0425 23:16:50.295651 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0425 23:16:54.731483 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0425 23:16:55.905154 3778 namespace.go:98] [trusted-artifact-signer] adding namespace I0425 23:16:55.905988 3778 namespace.go:102] [trusted-artifact-signer] adding namespace took 811.659µs I0425 23:16:55.932707 3778 namespace.go:142] [trusted-artifact-signer] updating namespace I0425 23:16:55.986503 3778 namespace.go:142] [trusted-artifact-signer] updating namespace W0425 23:16:56.696678 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0425 23:16:56.833727 3778 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-wn5cn] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-wn5cn for pod on switch ip-10-0-2-186.ec2.internal I0425 23:16:56.833825 3778 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-wn5cn I0425 23:16:56.894734 3778 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-wn5cn] pod update took 60.933025ms I0425 23:16:56.894764 3778 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-wn5cn] addLogicalPort annotation time took 60.967816ms I0425 23:16:56.896143 3778 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-wn5cn] addLogicalPort took 62.431581ms, libovsdb time 907.556µs I0425 23:16:57.209494 3778 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-wn5cn 25231e6be8e194d08abc2035487aaf250803fb90c3ff5a3064b9f5f7f396a248 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-wn5cn 25231e6be8e194d08abc2035487aaf250803fb90c3ff5a3064b9f5f7f396a248 network default NAD default NAD key ] I0425 23:16:57.219157 3778 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-wn5cn, hostIfaceName: 25231e6be8e194d, network: default, NAD default, SandboxID: "25231e6be8e194d08abc2035487aaf250803fb90c3ff5a3064b9f5f7f396a248", PCI device ID: , UID: "139a5464-895c-4f36-90c5-ab5e9e9c9b23", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0425 23:16:57.269267 3778 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-wn5cn 25231e6be8e194d08abc2035487aaf250803fb90c3ff5a3064b9f5f7f396a248 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-wn5cn 25231e6be8e194d08abc2035487aaf250803fb90c3ff5a3064b9f5f7f396a248 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:03:c7:d0:61:d2\",\"name\":\"25231e6be8e194d\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90a9d641-5390-487a-ae8f-f0831557b25c\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:16:59.220826 3778 namespace.go:142] [openshift-operators] updating namespace I0425 23:16:59.820136 3778 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-dwjmk] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-dwjmk for pod on switch ip-10-0-2-186.ec2.internal I0425 23:16:59.820230 3778 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-dwjmk I0425 23:16:59.845744 3778 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-dwjmk] pod update took 25.53254ms I0425 23:16:59.845774 3778 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-dwjmk] addLogicalPort annotation time took 25.565347ms I0425 23:16:59.847135 3778 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-dwjmk] addLogicalPort took 27.011623ms, libovsdb time 885.525µs I0425 23:17:00.178172 3778 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-dwjmk 487df16e0b07041786d601fd7c769581581faf6f3c1826e5544ca595e78a069e network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-dwjmk 487df16e0b07041786d601fd7c769581581faf6f3c1826e5544ca595e78a069e network default NAD default NAD key ] I0425 23:17:00.188719 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-dwjmk, hostIfaceName: 487df16e0b07041, network: default, NAD default, SandboxID: "487df16e0b07041786d601fd7c769581581faf6f3c1826e5544ca595e78a069e", PCI device ID: , UID: "64f74566-f7e0-4152-8e5a-6239a5af735a", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] W0425 23:17:00.226136 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0425 23:17:00.226159 3778 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0425 23:17:00.237965 3778 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-dwjmk 487df16e0b07041786d601fd7c769581581faf6f3c1826e5544ca595e78a069e network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-dwjmk 487df16e0b07041786d601fd7c769581581faf6f3c1826e5544ca595e78a069e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:e8:76:a5:a6:1f\",\"name\":\"487df16e0b07041\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8b653e48-6610-4b4a-a414-92c51abd4bd5\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:17:06.238241 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0425 23:17:06.238262 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0425 23:17:06.715465 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0425 23:17:10.126809 3778 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:17:10.127406 3778 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:17:10.140859 3778 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:17:10.140876 3778 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:17:10.146707 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0425 23:17:10.146730 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0425 23:17:15.971671 3778 namespace.go:98] [openshift-pipelines] adding namespace I0425 23:17:15.972487 3778 namespace.go:102] [openshift-pipelines] adding namespace took 791.775µs I0425 23:17:15.994640 3778 namespace.go:142] [openshift-pipelines] updating namespace I0425 23:17:16.819572 3778 namespace.go:142] [cert-manager] updating namespace I0425 23:17:17.020178 3778 namespace.go:142] [cert-manager-operator] updating namespace I0425 23:17:17.419356 3778 namespace.go:142] [konflux-operator] updating namespace I0425 23:17:17.620301 3778 namespace.go:142] [konflux-ui] updating namespace I0425 23:17:17.823701 3778 namespace.go:142] [open-cluster-management-2ptfd0doqkjckkqnviaqdhhsij5e416h] updating namespace I0425 23:17:18.224933 3778 namespace.go:142] [rhbk-operator] updating namespace I0425 23:17:18.425017 3778 namespace.go:142] [rhtpa-operator] updating namespace I0425 23:17:18.625369 3778 namespace.go:142] [trusted-artifact-signer] updating namespace I0425 23:17:18.824918 3778 namespace.go:142] [tsf] updating namespace I0425 23:17:19.025161 3778 namespace.go:142] [tsf-keycloak] updating namespace I0425 23:17:19.228058 3778 namespace.go:142] [tsf-tas] updating namespace I0425 23:17:19.428141 3778 namespace.go:142] [tsf-tpa] updating namespace I0425 23:17:19.556670 3778 namespace.go:142] [cert-manager] updating namespace I0425 23:17:19.566942 3778 namespace.go:142] [cert-manager-operator] updating namespace I0425 23:17:19.574504 3778 namespace.go:142] [default] updating namespace I0425 23:17:19.582986 3778 namespace.go:142] [konflux-operator] updating namespace I0425 23:17:19.592173 3778 namespace.go:142] [konflux-ui] updating namespace I0425 23:17:19.599246 3778 namespace.go:142] [open-cluster-management-2ptfd0doqkjckkqnviaqdhhsij5e416h] updating namespace I0425 23:17:19.609964 3778 namespace.go:142] [openshift] updating namespace I0425 23:17:19.617785 3778 namespace.go:142] [rhbk-operator] updating namespace I0425 23:17:19.627111 3778 namespace.go:142] [rhtpa-operator] updating namespace I0425 23:17:19.633907 3778 namespace.go:142] [trusted-artifact-signer] updating namespace I0425 23:17:19.651303 3778 namespace.go:142] [tsf] updating namespace I0425 23:17:19.670266 3778 namespace.go:142] [tsf-keycloak] updating namespace I0425 23:17:19.690200 3778 namespace.go:142] [tsf-tas] updating namespace I0425 23:17:19.713027 3778 namespace.go:142] [tsf-tpa] updating namespace I0425 23:17:19.810572 3778 namespace.go:142] [cert-manager] updating namespace I0425 23:17:19.911323 3778 namespace.go:142] [cert-manager-operator] updating namespace I0425 23:17:20.012298 3778 namespace.go:142] [default] updating namespace I0425 23:17:20.111495 3778 namespace.go:142] [konflux-operator] updating namespace I0425 23:17:20.211392 3778 namespace.go:142] [konflux-ui] updating namespace I0425 23:17:20.313052 3778 namespace.go:142] [open-cluster-management-2ptfd0doqkjckkqnviaqdhhsij5e416h] updating namespace I0425 23:17:20.409691 3778 namespace.go:142] [openshift] updating namespace I0425 23:17:20.511025 3778 namespace.go:142] [rhbk-operator] updating namespace I0425 23:17:20.612342 3778 namespace.go:142] [rhtpa-operator] updating namespace I0425 23:17:20.710249 3778 namespace.go:142] [trusted-artifact-signer] updating namespace I0425 23:17:20.810712 3778 namespace.go:142] [tsf] updating namespace I0425 23:17:20.912270 3778 namespace.go:142] [tsf-keycloak] updating namespace I0425 23:17:21.012940 3778 namespace.go:142] [tsf-tas] updating namespace I0425 23:17:21.112152 3778 namespace.go:142] [tsf-tpa] updating namespace I0425 23:17:21.377599 3778 namespace.go:142] [openshift-pipelines] updating namespace W0425 23:17:26.067023 3778 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0425 23:17:26.086546 3778 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0425 23:17:30.006739 3778 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-z2nk2] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-z2nk2 for pod on switch ip-10-0-2-186.ec2.internal I0425 23:17:30.006845 3778 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-z2nk2 I0425 23:17:30.022492 3778 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-z2nk2] pod update took 15.675921ms I0425 23:17:30.022515 3778 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-z2nk2] addLogicalPort annotation time took 15.704776ms I0425 23:17:30.023684 3778 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-z2nk2] addLogicalPort took 16.965328ms, libovsdb time 828.443µs I0425 23:17:30.092684 3778 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-9d9jk] creating logical port openshift-pipelines_tekton-events-controller-57d7f78b8d-9d9jk for pod on switch ip-10-0-2-186.ec2.internal I0425 23:17:30.092776 3778 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-57d7f78b8d-9d9jk I0425 23:17:30.112364 3778 pod.go:62] [openshift-pipelines/tekton-events-controller-57d7f78b8d-9d9jk] pod update took 19.613429ms I0425 23:17:30.112388 3778 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-9d9jk] addLogicalPort annotation time took 19.643548ms I0425 23:17:30.113664 3778 pods.go:271] [openshift-pipelines/tekton-events-controller-57d7f78b8d-9d9jk] addLogicalPort took 20.992137ms, libovsdb time 916.07µs I0425 23:17:30.379300 3778 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-z2nk2 b64ac97277e4c89780e54f706fc145461d16ee271d712fe175e8eb671cfb7ac3 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-z2nk2 b64ac97277e4c89780e54f706fc145461d16ee271d712fe175e8eb671cfb7ac3 network default NAD default NAD key ] I0425 23:17:30.388942 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-z2nk2, hostIfaceName: b64ac97277e4c89, network: default, NAD default, SandboxID: "b64ac97277e4c89780e54f706fc145461d16ee271d712fe175e8eb671cfb7ac3", PCI device ID: , UID: "585829ab-171c-4747-a0af-e0bb1d0950de", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0425 23:17:30.441561 3778 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-z2nk2 b64ac97277e4c89780e54f706fc145461d16ee271d712fe175e8eb671cfb7ac3 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-z2nk2 b64ac97277e4c89780e54f706fc145461d16ee271d712fe175e8eb671cfb7ac3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:d3:a8:79:ad:9a\",\"name\":\"b64ac97277e4c89\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/554dca66-0e89-40aa-971a-1353d3168dd6\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:17:30.453493 3778 cni.go:452] [openshift-pipelines/tekton-events-controller-57d7f78b8d-9d9jk b71f28066d4410b146b6ca84347e4e9f187a05fd1cd7866b99d4bff12a98956c network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-9d9jk b71f28066d4410b146b6ca84347e4e9f187a05fd1cd7866b99d4bff12a98956c network default NAD default NAD key ] I0425 23:17:30.463062 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-57d7f78b8d-9d9jk, hostIfaceName: b71f28066d4410b, network: default, NAD default, SandboxID: "b71f28066d4410b146b6ca84347e4e9f187a05fd1cd7866b99d4bff12a98956c", PCI device ID: , UID: "27dfa76e-ca37-42f7-8fa8-7d81b4d81ea6", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0425 23:17:30.514538 3778 cni.go:473] [openshift-pipelines/tekton-events-controller-57d7f78b8d-9d9jk b71f28066d4410b146b6ca84347e4e9f187a05fd1cd7866b99d4bff12a98956c network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-9d9jk b71f28066d4410b146b6ca84347e4e9f187a05fd1cd7866b99d4bff12a98956c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:54:1a:d9:15:bb\",\"name\":\"b71f28066d4410b\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cd347b2f-ff21-4c43-b421-6d400c3d38a8\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:17:33.381572 3778 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0425 23:17:33.381597 3778 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0425 23:17:33.381604 3778 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0425 23:17:33.629293 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=31 W0425 23:17:34.119051 3778 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0425 23:17:34.119071 3778 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0425 23:17:34.119076 3778 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0425 23:17:34.119080 3778 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0425 23:17:34.385656 3778 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0425 23:17:34.385677 3778 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0425 23:17:34.385681 3778 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0425 23:17:34.641050 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=49 W0425 23:17:44.386761 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0425 23:17:44.386783 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0425 23:17:44.386788 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0425 23:17:45.393718 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0425 23:17:45.393739 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0425 23:17:45.393745 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0425 23:17:56.324410 3778 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-ffgx2] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-ffgx2 for pod on switch ip-10-0-2-186.ec2.internal I0425 23:17:56.324502 3778 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-ffgx2 I0425 23:17:56.339994 3778 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-ffgx2] pod update took 15.500813ms I0425 23:17:56.340029 3778 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-ffgx2] addLogicalPort annotation time took 15.551876ms I0425 23:17:56.341420 3778 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-ffgx2] addLogicalPort took 17.029972ms, libovsdb time 918.972µs I0425 23:17:56.683554 3778 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-ffgx2 5d159ee38305bba48a489e605fcd2b8951f9bcab69ecc1bd14240d5576c9afce network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-ffgx2 5d159ee38305bba48a489e605fcd2b8951f9bcab69ecc1bd14240d5576c9afce network default NAD default NAD key ] I0425 23:17:56.693710 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-ffgx2, hostIfaceName: 5d159ee38305bba, network: default, NAD default, SandboxID: "5d159ee38305bba48a489e605fcd2b8951f9bcab69ecc1bd14240d5576c9afce", PCI device ID: , UID: "f98b218b-f4d6-4fd1-a949-05a92ea2c67b", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0425 23:17:56.746712 3778 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-ffgx2 5d159ee38305bba48a489e605fcd2b8951f9bcab69ecc1bd14240d5576c9afce network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-ffgx2 5d159ee38305bba48a489e605fcd2b8951f9bcab69ecc1bd14240d5576c9afce network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:2c:a7:eb:c9:ad\",\"name\":\"5d159ee38305bba\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a9f14f53-3b61-43d1-a441-ca4928465774\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:17:59.963478 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0425 23:18:02.377612 3778 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:18:02.378168 3778 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:18:02.397670 3778 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:18:02.397698 3778 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:18:02.400264 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0425 23:18:02.400282 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0425 23:18:04.284986 3778 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0425 23:18:04.285486 3778 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0425 23:18:04.285509 3778 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0425 23:18:04.287510 3778 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0425 23:18:10.297833 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0425 23:18:10.297858 3778 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0425 23:18:10.297834 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0425 23:18:10.297906 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0425 23:18:12.808334 3778 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-55fb698565-56lpv] creating logical port openshift-pipelines_tekton-chains-controller-55fb698565-56lpv for pod on switch ip-10-0-2-186.ec2.internal I0425 23:18:12.808437 3778 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-55fb698565-56lpv I0425 23:18:12.822379 3778 pod.go:62] [openshift-pipelines/tekton-chains-controller-55fb698565-56lpv] pod update took 13.97054ms I0425 23:18:12.822400 3778 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-chains-controller-55fb698565-56lpv] addLogicalPort annotation time took 13.996694ms I0425 23:18:12.823708 3778 pods.go:271] [openshift-pipelines/tekton-chains-controller-55fb698565-56lpv] addLogicalPort took 15.390343ms, libovsdb time 975.903µs I0425 23:18:13.167938 3778 cni.go:452] [openshift-pipelines/tekton-chains-controller-55fb698565-56lpv ddbe0f62ebba1aa1b776055815260927c189fce9c654609857d0277bc2c2cbc5 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-56lpv ddbe0f62ebba1aa1b776055815260927c189fce9c654609857d0277bc2c2cbc5 network default NAD default NAD key ] I0425 23:18:13.178326 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-55fb698565-56lpv, hostIfaceName: ddbe0f62ebba1aa, network: default, NAD default, SandboxID: "ddbe0f62ebba1aa1b776055815260927c189fce9c654609857d0277bc2c2cbc5", PCI device ID: , UID: "fb5e41f3-ba77-4046-9a6b-c8962d1e2ae3", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0425 23:18:13.232466 3778 cni.go:473] [openshift-pipelines/tekton-chains-controller-55fb698565-56lpv ddbe0f62ebba1aa1b776055815260927c189fce9c654609857d0277bc2c2cbc5 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-56lpv ddbe0f62ebba1aa1b776055815260927c189fce9c654609857d0277bc2c2cbc5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:b2:3d:98:df:58\",\"name\":\"ddbe0f62ebba1aa\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e8d48638-a355-482d-bf22-d5237f1d53e3\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:18:17.503461 3778 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0425 23:18:37.604577 3778 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:18:37.605087 3778 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:18:37.649830 3778 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:18:37.649854 3778 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:18:37.654771 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0425 23:18:37.654795 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0425 23:18:37.736309 3778 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-68bhj] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-68bhj for pod on switch ip-10-0-2-186.ec2.internal I0425 23:18:37.736407 3778 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-68bhj I0425 23:18:37.752958 3778 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-68bhj] pod update took 16.578696ms I0425 23:18:37.752982 3778 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-68bhj] addLogicalPort annotation time took 16.609137ms I0425 23:18:37.754246 3778 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-68bhj] addLogicalPort took 17.954168ms, libovsdb time 834.356µs I0425 23:18:38.096946 3778 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-68bhj d6bcd2c645c5afb01be6c2feaa98cb810523d083f21e7200b4126af07787bdc2 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-68bhj d6bcd2c645c5afb01be6c2feaa98cb810523d083f21e7200b4126af07787bdc2 network default NAD default NAD key ] I0425 23:18:38.107015 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-68bhj, hostIfaceName: d6bcd2c645c5afb, network: default, NAD default, SandboxID: "d6bcd2c645c5afb01be6c2feaa98cb810523d083f21e7200b4126af07787bdc2", PCI device ID: , UID: "ab44531c-5405-4acd-93a2-70361a2fada0", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0425 23:18:38.160611 3778 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-68bhj d6bcd2c645c5afb01be6c2feaa98cb810523d083f21e7200b4126af07787bdc2 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-68bhj d6bcd2c645c5afb01be6c2feaa98cb810523d083f21e7200b4126af07787bdc2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:5b:3c:b2:94:46\",\"name\":\"d6bcd2c645c5afb\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3cad7a83-ca8a-4bc3-b37c-0ddeab07d8e0\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:18:39.508804 3778 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-flftg-zsllw] creating logical port tsf-tas_segment-backup-installation-flftg-zsllw for pod on switch ip-10-0-2-186.ec2.internal I0425 23:18:39.508914 3778 kube.go:256] Updating pod tsf-tas/segment-backup-installation-flftg-zsllw I0425 23:18:39.523327 3778 pod.go:62] [tsf-tas/segment-backup-installation-flftg-zsllw] pod update took 14.440987ms I0425 23:18:39.523354 3778 base_network_controller_pods.go:951] [default/tsf-tas/segment-backup-installation-flftg-zsllw] addLogicalPort annotation time took 14.475352ms I0425 23:18:39.524586 3778 pods.go:271] [tsf-tas/segment-backup-installation-flftg-zsllw] addLogicalPort took 15.805439ms, libovsdb time 861.019µs I0425 23:18:39.873672 3778 cni.go:452] [tsf-tas/segment-backup-installation-flftg-zsllw d9cf48ff68e3d2a358ba903ce09d9c46ea42cbdbfbdb673eaeec9e3843644a22 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/segment-backup-installation-flftg-zsllw d9cf48ff68e3d2a358ba903ce09d9c46ea42cbdbfbdb673eaeec9e3843644a22 network default NAD default NAD key ] I0425 23:18:39.883841 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-flftg-zsllw, hostIfaceName: d9cf48ff68e3d2a, network: default, NAD default, SandboxID: "d9cf48ff68e3d2a358ba903ce09d9c46ea42cbdbfbdb673eaeec9e3843644a22", PCI device ID: , UID: "be8527cc-52c7-4335-a5f9-70b930ccf6e0", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0425 23:18:39.941255 3778 cni.go:473] [tsf-tas/segment-backup-installation-flftg-zsllw d9cf48ff68e3d2a358ba903ce09d9c46ea42cbdbfbdb673eaeec9e3843644a22 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/segment-backup-installation-flftg-zsllw d9cf48ff68e3d2a358ba903ce09d9c46ea42cbdbfbdb673eaeec9e3843644a22 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:1e:cb:92:5c:f9\",\"name\":\"d9cf48ff68e3d2a\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f99741e1-a271-467e-b9c4-222684cf6daf\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:18:40.344631 3778 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-fmccw-vf76d] creating logical port tsf-tas_ctlog-createtree-job-fmccw-vf76d for pod on switch ip-10-0-2-186.ec2.internal I0425 23:18:40.344739 3778 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-fmccw-vf76d I0425 23:18:40.362729 3778 pod.go:62] [tsf-tas/ctlog-createtree-job-fmccw-vf76d] pod update took 18.022542ms I0425 23:18:40.362753 3778 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-createtree-job-fmccw-vf76d] addLogicalPort annotation time took 18.054855ms I0425 23:18:40.363993 3778 pods.go:271] [tsf-tas/ctlog-createtree-job-fmccw-vf76d] addLogicalPort took 19.367303ms, libovsdb time 833.295µs I0425 23:18:40.676319 3778 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-qvf82-7rtgp] creating logical port tsf-tas_rekor-createtree-job-qvf82-7rtgp for pod on switch ip-10-0-2-186.ec2.internal I0425 23:18:40.676493 3778 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-qvf82-7rtgp I0425 23:18:40.690132 3778 pod.go:62] [tsf-tas/rekor-createtree-job-qvf82-7rtgp] pod update took 13.669106ms I0425 23:18:40.690216 3778 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-qvf82-7rtgp] addLogicalPort annotation time took 13.758649ms I0425 23:18:40.691977 3778 pods.go:271] [tsf-tas/rekor-createtree-job-qvf82-7rtgp] addLogicalPort took 15.676973ms, libovsdb time 1.258879ms I0425 23:18:40.726685 3778 cni.go:452] [tsf-tas/ctlog-createtree-job-fmccw-vf76d 4cbe8f2e8996b7d4e28f7ec61d4673570a094edf0724ae5ef0d65a6c1125a258 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-fmccw-vf76d 4cbe8f2e8996b7d4e28f7ec61d4673570a094edf0724ae5ef0d65a6c1125a258 network default NAD default NAD key ] I0425 23:18:40.738452 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-fmccw-vf76d, hostIfaceName: 4cbe8f2e8996b7d, network: default, NAD default, SandboxID: "4cbe8f2e8996b7d4e28f7ec61d4673570a094edf0724ae5ef0d65a6c1125a258", PCI device ID: , UID: "fa8c68ca-1595-402f-87bd-933f07353f11", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0425 23:18:40.812586 3778 cni.go:473] [tsf-tas/ctlog-createtree-job-fmccw-vf76d 4cbe8f2e8996b7d4e28f7ec61d4673570a094edf0724ae5ef0d65a6c1125a258 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-fmccw-vf76d 4cbe8f2e8996b7d4e28f7ec61d4673570a094edf0724ae5ef0d65a6c1125a258 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:d1:68:20:34:ba\",\"name\":\"4cbe8f2e8996b7d\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb759c31-03e7-4dc8-a87c-27143be07809\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:18:41.889984 3778 cni.go:452] [tsf-tas/rekor-createtree-job-qvf82-7rtgp 4c5b61be4fcc470d71112013e9b7d06655ab1f04f1fcb5b768e81726d438b903 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-qvf82-7rtgp 4c5b61be4fcc470d71112013e9b7d06655ab1f04f1fcb5b768e81726d438b903 network default NAD default NAD key ] I0425 23:18:41.902109 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-qvf82-7rtgp, hostIfaceName: 4c5b61be4fcc470, network: default, NAD default, SandboxID: "4c5b61be4fcc470d71112013e9b7d06655ab1f04f1fcb5b768e81726d438b903", PCI device ID: , UID: "962890d4-7f57-42e1-8fe2-509aabce0cec", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0425 23:18:41.969143 3778 cni.go:473] [tsf-tas/rekor-createtree-job-qvf82-7rtgp 4c5b61be4fcc470d71112013e9b7d06655ab1f04f1fcb5b768e81726d438b903 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-qvf82-7rtgp 4c5b61be4fcc470d71112013e9b7d06655ab1f04f1fcb5b768e81726d438b903 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:35:65:f6:f0:a0\",\"name\":\"4c5b61be4fcc470\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1b50937b-7906-4a75-968f-a7d587886354\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:18:42.212941 3778 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0425 23:18:42.212971 3778 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0425 23:18:42.212980 3778 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0425 23:18:43.208464 3778 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-d4547c9bc-4rkk8] creating logical port tsf-tas_trillian-db-d4547c9bc-4rkk8 for pod on switch ip-10-0-2-186.ec2.internal I0425 23:18:43.208662 3778 kube.go:256] Updating pod tsf-tas/trillian-db-d4547c9bc-4rkk8 I0425 23:18:43.221808 3778 pod.go:62] [tsf-tas/trillian-db-d4547c9bc-4rkk8] pod update took 13.172804ms I0425 23:18:43.221843 3778 base_network_controller_pods.go:951] [default/tsf-tas/trillian-db-d4547c9bc-4rkk8] addLogicalPort annotation time took 13.213465ms I0425 23:18:43.223585 3778 pods.go:271] [tsf-tas/trillian-db-d4547c9bc-4rkk8] addLogicalPort took 15.140987ms, libovsdb time 1.156173ms W0425 23:18:44.232456 3778 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0425 23:18:44.232475 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0425 23:18:48.262036 3778 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0425 23:18:48.303644 3778 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0425 23:18:49.666750 3778 cni.go:452] [tsf-tas/trillian-db-d4547c9bc-4rkk8 436b5a52caec949adefac9ff3d187c5c5fbaec200fe1a9b604bee7c9de4e4d7d network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-db-d4547c9bc-4rkk8 436b5a52caec949adefac9ff3d187c5c5fbaec200fe1a9b604bee7c9de4e4d7d network default NAD default NAD key ] I0425 23:18:49.677310 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-d4547c9bc-4rkk8, hostIfaceName: 436b5a52caec949, network: default, NAD default, SandboxID: "436b5a52caec949adefac9ff3d187c5c5fbaec200fe1a9b604bee7c9de4e4d7d", PCI device ID: , UID: "94bddddb-5fa9-45bb-bf48-8aea30e629a1", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0425 23:18:49.736185 3778 cni.go:473] [tsf-tas/trillian-db-d4547c9bc-4rkk8 436b5a52caec949adefac9ff3d187c5c5fbaec200fe1a9b604bee7c9de4e4d7d network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-db-d4547c9bc-4rkk8 436b5a52caec949adefac9ff3d187c5c5fbaec200fe1a9b604bee7c9de4e4d7d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:7f:0b:70:8b:0b\",\"name\":\"436b5a52caec949\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/27ca9a11-4be3-48cf-8179-87ddd1a2c32c\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:18:52.693121 3778 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-9hgx4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:18:52.693752 3778 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-9hgx4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0425 23:18:52.790372 3778 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0425 23:18:53.593487 3778 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-9hgx4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:18:53.593521 3778 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-9hgx4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:18:53.780861 3778 cni.go:452] [tsf-tas/segment-backup-installation-flftg-zsllw d9cf48ff68e3d2a358ba903ce09d9c46ea42cbdbfbdb673eaeec9e3843644a22 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/segment-backup-installation-flftg-zsllw d9cf48ff68e3d2a358ba903ce09d9c46ea42cbdbfbdb673eaeec9e3843644a22 network default NAD default NAD key ] I0425 23:18:53.873045 3778 cni.go:473] [tsf-tas/segment-backup-installation-flftg-zsllw d9cf48ff68e3d2a358ba903ce09d9c46ea42cbdbfbdb673eaeec9e3843644a22 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/segment-backup-installation-flftg-zsllw d9cf48ff68e3d2a358ba903ce09d9c46ea42cbdbfbdb673eaeec9e3843644a22 network default NAD default NAD key default], result "{}", err I0425 23:18:53.920727 3778 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-flftg-zsllw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:18:53.920769 3778 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-flftg-zsllw I0425 23:18:53.922006 3778 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-flftg-zsllw, ips: 10.128.0.34 I0425 23:18:53.922036 3778 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-flftg-zsllw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:18:54.785733 3778 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-flftg-zsllw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:18:54.785755 3778 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-flftg-zsllw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0425 23:19:03.815000 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql I0425 23:19:07.426371 3778 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-ck4mt] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-ck4mt for pod on switch ip-10-0-2-186.ec2.internal I0425 23:19:07.426470 3778 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-ck4mt I0425 23:19:07.440243 3778 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-ck4mt] pod update took 13.798019ms I0425 23:19:07.440272 3778 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-ck4mt] addLogicalPort annotation time took 13.830819ms I0425 23:19:07.441684 3778 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-ck4mt] addLogicalPort took 15.336187ms, libovsdb time 900.502µs I0425 23:19:07.785001 3778 cni.go:452] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-ck4mt 168f6ab56c3742d649daf8bbe4024d024e71416e2b0c54768b6e2eca5495c1c6 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-ck4mt 168f6ab56c3742d649daf8bbe4024d024e71416e2b0c54768b6e2eca5495c1c6 network default NAD default NAD key ] I0425 23:19:07.795750 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-ck4mt, hostIfaceName: 168f6ab56c3742d, network: default, NAD default, SandboxID: "168f6ab56c3742d649daf8bbe4024d024e71416e2b0c54768b6e2eca5495c1c6", PCI device ID: , UID: "ad592809-f89e-4b95-a5c2-b7c0433de6ff", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0425 23:19:07.852213 3778 cni.go:473] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-ck4mt 168f6ab56c3742d649daf8bbe4024d024e71416e2b0c54768b6e2eca5495c1c6 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-ck4mt 168f6ab56c3742d649daf8bbe4024d024e71416e2b0c54768b6e2eca5495c1c6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:d5:d1:70:e4:3a\",\"name\":\"168f6ab56c3742d\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3a80bc51-2ef4-4f8b-98e4-eb0232d847e9\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:19:10.669089 3778 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0425 23:19:10.694350 3778 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0425 23:19:10.694378 3778 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0425 23:19:18.738524 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0425 23:19:25.714799 3778 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0425 23:19:28.905532 3778 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:28.906113 3778 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:28.924891 3778 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:28.924914 3778 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:28.929772 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0425 23:19:28.929806 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0425 23:19:37.961934 3778 obj_retry.go:498] Detected object tsf/patch-tekton-config-stgd9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:37.962519 3778 obj_retry.go:498] Detected object tsf/patch-tekton-config-stgd9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:38.835302 3778 obj_retry.go:492] Detected object tsf/patch-tekton-config-stgd9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:38.835326 3778 obj_retry.go:492] Detected object tsf/patch-tekton-config-stgd9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:39.011049 3778 cni.go:452] [tsf-tas/ctlog-createtree-job-fmccw-vf76d 4cbe8f2e8996b7d4e28f7ec61d4673570a094edf0724ae5ef0d65a6c1125a258 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-fmccw-vf76d 4cbe8f2e8996b7d4e28f7ec61d4673570a094edf0724ae5ef0d65a6c1125a258 network default NAD default NAD key ] I0425 23:19:39.105312 3778 cni.go:473] [tsf-tas/ctlog-createtree-job-fmccw-vf76d 4cbe8f2e8996b7d4e28f7ec61d4673570a094edf0724ae5ef0d65a6c1125a258 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-fmccw-vf76d 4cbe8f2e8996b7d4e28f7ec61d4673570a094edf0724ae5ef0d65a6c1125a258 network default NAD default NAD key default], result "{}", err I0425 23:19:39.152009 3778 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-fmccw-vf76d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:39.152038 3778 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-fmccw-vf76d I0425 23:19:39.153299 3778 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-fmccw-vf76d, ips: 10.128.0.35 I0425 23:19:39.153328 3778 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-fmccw-vf76d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:40.024513 3778 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-fmccw-vf76d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:40.024546 3778 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-fmccw-vf76d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:44.634648 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=412 W0425 23:19:44.854088 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0425 23:19:44.998090 3778 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-2tstr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:44.998668 3778 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-2tstr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:46.017393 3778 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-2tstr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:46.017422 3778 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-2tstr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:46.050505 3778 obj_retry.go:492] Detected object tsf/patch-tekton-config-stgd9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:46.050525 3778 obj_retry.go:492] Detected object tsf/patch-tekton-config-stgd9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:46.057025 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-stgd9 I0425 23:19:46.057053 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-stgd9 I0425 23:19:48.057210 3778 cni.go:452] [tsf-tas/rekor-createtree-job-qvf82-7rtgp 4c5b61be4fcc470d71112013e9b7d06655ab1f04f1fcb5b768e81726d438b903 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-qvf82-7rtgp 4c5b61be4fcc470d71112013e9b7d06655ab1f04f1fcb5b768e81726d438b903 network default NAD default NAD key ] I0425 23:19:48.153777 3778 cni.go:473] [tsf-tas/rekor-createtree-job-qvf82-7rtgp 4c5b61be4fcc470d71112013e9b7d06655ab1f04f1fcb5b768e81726d438b903 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-qvf82-7rtgp 4c5b61be4fcc470d71112013e9b7d06655ab1f04f1fcb5b768e81726d438b903 network default NAD default NAD key default], result "{}", err I0425 23:19:48.203573 3778 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-qvf82-7rtgp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:48.203604 3778 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-qvf82-7rtgp I0425 23:19:48.204943 3778 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-qvf82-7rtgp, ips: 10.128.0.36 I0425 23:19:48.204973 3778 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-qvf82-7rtgp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:49.068500 3778 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-qvf82-7rtgp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:49.068534 3778 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-qvf82-7rtgp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:51.033346 3778 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:51.033912 3778 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:51.052591 3778 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:51.052619 3778 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:51.058041 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0425 23:19:51.058066 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0425 23:19:51.820195 3778 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-89464c768-kfqkz] creating logical port tsf-tas_rekor-search-ui-89464c768-kfqkz for pod on switch ip-10-0-2-186.ec2.internal I0425 23:19:51.820293 3778 kube.go:256] Updating pod tsf-tas/rekor-search-ui-89464c768-kfqkz I0425 23:19:51.833593 3778 pod.go:62] [tsf-tas/rekor-search-ui-89464c768-kfqkz] pod update took 13.323144ms I0425 23:19:51.833628 3778 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-89464c768-kfqkz] addLogicalPort annotation time took 13.366047ms I0425 23:19:51.835479 3778 pods.go:271] [tsf-tas/rekor-search-ui-89464c768-kfqkz] addLogicalPort took 15.295904ms, libovsdb time 1.231138ms I0425 23:19:52.182745 3778 cni.go:452] [tsf-tas/rekor-search-ui-89464c768-kfqkz 36191e64b213b0669313c75180eceee26787321cb540b0ce700a4f101e9296f1 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-89464c768-kfqkz 36191e64b213b0669313c75180eceee26787321cb540b0ce700a4f101e9296f1 network default NAD default NAD key ] I0425 23:19:52.193049 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-89464c768-kfqkz, hostIfaceName: 36191e64b213b06, network: default, NAD default, SandboxID: "36191e64b213b0669313c75180eceee26787321cb540b0ce700a4f101e9296f1", PCI device ID: , UID: "a0601392-4f6f-4097-a6b4-e9db54f6ff71", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0425 23:19:52.250245 3778 cni.go:473] [tsf-tas/rekor-search-ui-89464c768-kfqkz 36191e64b213b0669313c75180eceee26787321cb540b0ce700a4f101e9296f1 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-89464c768-kfqkz 36191e64b213b0669313c75180eceee26787321cb540b0ce700a4f101e9296f1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:82:99:e6:d1:b5\",\"name\":\"36191e64b213b06\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/21962c55-d5d8-49cc-b066-205fc51af1ff\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:19:52.468249 3778 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-5284v] creating logical port tsf-tpa_create-db-5284v for pod on switch ip-10-0-2-186.ec2.internal I0425 23:19:52.468395 3778 kube.go:256] Updating pod tsf-tpa/create-db-5284v I0425 23:19:52.481380 3778 pod.go:62] [tsf-tpa/create-db-5284v] pod update took 13.004261ms I0425 23:19:52.481408 3778 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-5284v] addLogicalPort annotation time took 13.039592ms I0425 23:19:52.482798 3778 pods.go:271] [tsf-tpa/create-db-5284v] addLogicalPort took 14.568062ms, libovsdb time 1.002524ms I0425 23:19:52.832719 3778 cni.go:452] [tsf-tpa/create-db-5284v 6184f95c924fb4c164c070b6d67c1f53896131005150f3db4186971dc584863e network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-5284v 6184f95c924fb4c164c070b6d67c1f53896131005150f3db4186971dc584863e network default NAD default NAD key ] I0425 23:19:52.842631 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-5284v, hostIfaceName: 6184f95c924fb4c, network: default, NAD default, SandboxID: "6184f95c924fb4c164c070b6d67c1f53896131005150f3db4186971dc584863e", PCI device ID: , UID: "e2f5fca6-0c48-4ede-8dc0-b25a08ef5931", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0425 23:19:52.897359 3778 cni.go:473] [tsf-tpa/create-db-5284v 6184f95c924fb4c164c070b6d67c1f53896131005150f3db4186971dc584863e network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-5284v 6184f95c924fb4c164c070b6d67c1f53896131005150f3db4186971dc584863e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:c6:3c:66:00:81\",\"name\":\"6184f95c924fb4c\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f45652ea-e58b-4fe8-aeac-14755e125652\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:19:54.420535 3778 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-59977cd7b5-6kf4v] creating logical port tsf-tas_rekor-server-59977cd7b5-6kf4v for pod on switch ip-10-0-2-186.ec2.internal I0425 23:19:54.420637 3778 kube.go:256] Updating pod tsf-tas/rekor-server-59977cd7b5-6kf4v I0425 23:19:54.433442 3778 pod.go:62] [tsf-tas/rekor-server-59977cd7b5-6kf4v] pod update took 12.830795ms I0425 23:19:54.433472 3778 base_network_controller_pods.go:951] [default/tsf-tas/rekor-server-59977cd7b5-6kf4v] addLogicalPort annotation time took 12.864744ms I0425 23:19:54.435075 3778 pods.go:271] [tsf-tas/rekor-server-59977cd7b5-6kf4v] addLogicalPort took 14.564997ms, libovsdb time 1.061571ms W0425 23:19:56.096067 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0425 23:19:56.677860 3778 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0425 23:19:57.110578 3778 cni.go:452] [tsf-tpa/create-db-5284v 6184f95c924fb4c164c070b6d67c1f53896131005150f3db4186971dc584863e network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-5284v 6184f95c924fb4c164c070b6d67c1f53896131005150f3db4186971dc584863e network default NAD default NAD key ] I0425 23:19:57.203518 3778 cni.go:473] [tsf-tpa/create-db-5284v 6184f95c924fb4c164c070b6d67c1f53896131005150f3db4186971dc584863e network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-5284v 6184f95c924fb4c164c070b6d67c1f53896131005150f3db4186971dc584863e network default NAD default NAD key default], result "{}", err I0425 23:19:57.256509 3778 obj_retry.go:498] Detected object tsf-tpa/create-db-5284v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:57.256540 3778 pods.go:176] Deleting pod: tsf-tpa/create-db-5284v I0425 23:19:57.257938 3778 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-5284v, ips: 10.128.0.40 I0425 23:19:57.257968 3778 obj_retry.go:498] Detected object tsf-tpa/create-db-5284v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:19:58.279653 3778 obj_retry.go:492] Detected object tsf-tpa/create-db-5284v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:58.279678 3778 obj_retry.go:492] Detected object tsf-tpa/create-db-5284v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:19:58.379723 3778 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-jzmgk] creating logical port tsf-tpa_migrate-db-jzmgk for pod on switch ip-10-0-2-186.ec2.internal I0425 23:19:58.379811 3778 kube.go:256] Updating pod tsf-tpa/migrate-db-jzmgk I0425 23:19:58.392855 3778 pod.go:62] [tsf-tpa/migrate-db-jzmgk] pod update took 13.066144ms I0425 23:19:58.392886 3778 base_network_controller_pods.go:951] [default/tsf-tpa/migrate-db-jzmgk] addLogicalPort annotation time took 13.100126ms I0425 23:19:58.394388 3778 pods.go:271] [tsf-tpa/migrate-db-jzmgk] addLogicalPort took 14.677276ms, libovsdb time 1.078142ms I0425 23:19:58.684502 3778 cni.go:452] [tsf-tas/rekor-server-59977cd7b5-6kf4v 4b1a0ac5df720b3749e45b3a7509555733ff20a3b87f4e3d7e78ca4dba1f161b network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-59977cd7b5-6kf4v 4b1a0ac5df720b3749e45b3a7509555733ff20a3b87f4e3d7e78ca4dba1f161b network default NAD default NAD key ] I0425 23:19:58.694771 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-59977cd7b5-6kf4v, hostIfaceName: 4b1a0ac5df720b3, network: default, NAD default, SandboxID: "4b1a0ac5df720b3749e45b3a7509555733ff20a3b87f4e3d7e78ca4dba1f161b", PCI device ID: , UID: "b9828b58-f1eb-41bf-827e-36e9d806ce4b", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0425 23:19:58.744774 3778 cni.go:452] [tsf-tpa/migrate-db-jzmgk 572c41526f06901c72208da7f3b96264c311c9fdd066bd107f405bff2a1ec03b network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/migrate-db-jzmgk 572c41526f06901c72208da7f3b96264c311c9fdd066bd107f405bff2a1ec03b network default NAD default NAD key ] I0425 23:19:58.754699 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-jzmgk, hostIfaceName: 572c41526f06901, network: default, NAD default, SandboxID: "572c41526f06901c72208da7f3b96264c311c9fdd066bd107f405bff2a1ec03b", PCI device ID: , UID: "cc0986da-326b-4304-bde4-e5083d9d541b", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0425 23:19:58.756030 3778 cni.go:473] [tsf-tas/rekor-server-59977cd7b5-6kf4v 4b1a0ac5df720b3749e45b3a7509555733ff20a3b87f4e3d7e78ca4dba1f161b network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-59977cd7b5-6kf4v 4b1a0ac5df720b3749e45b3a7509555733ff20a3b87f4e3d7e78ca4dba1f161b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:e5:cf:c5:f3:ea\",\"name\":\"4b1a0ac5df720b3\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5c03a75b-b2ab-4f4c-b487-bbd0003b983a\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:19:58.814349 3778 cni.go:473] [tsf-tpa/migrate-db-jzmgk 572c41526f06901c72208da7f3b96264c311c9fdd066bd107f405bff2a1ec03b network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/migrate-db-jzmgk 572c41526f06901c72208da7f3b96264c311c9fdd066bd107f405bff2a1ec03b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:df:9c:f7:e2:34\",\"name\":\"572c41526f06901\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/042b0fb5-f719-4ade-8250-a89925508762\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:20:00.165190 3778 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29619320-rth5x for pod on switch ip-10-0-2-186.ec2.internal I0425 23:20:00.165319 3778 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x I0425 23:20:00.177588 3778 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x] pod update took 12.301009ms I0425 23:20:00.177618 3778 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x] addLogicalPort annotation time took 12.360487ms I0425 23:20:00.178971 3778 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x] addLogicalPort took 13.800219ms, libovsdb time 868.221µs I0425 23:20:00.527514 3778 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x bce44a378894cb6cebc0f29ea6612eb2d9178d511d84aba0f5b45f4453616bc8 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x bce44a378894cb6cebc0f29ea6612eb2d9178d511d84aba0f5b45f4453616bc8 network default NAD default NAD key ] I0425 23:20:00.538528 3778 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29619320-rth5x, hostIfaceName: bce44a378894cb6, network: default, NAD default, SandboxID: "bce44a378894cb6cebc0f29ea6612eb2d9178d511d84aba0f5b45f4453616bc8", PCI device ID: , UID: "bd29a001-f339-4199-a224-ad41bdadba85", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0425 23:20:00.600071 3778 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x bce44a378894cb6cebc0f29ea6612eb2d9178d511d84aba0f5b45f4453616bc8 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x bce44a378894cb6cebc0f29ea6612eb2d9178d511d84aba0f5b45f4453616bc8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:c4:3c:5d:9f:90\",\"name\":\"bce44a378894cb6\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9fb67f53-809e-4a81-a2f2-54cb4a75c804\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:20:01.135257 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0425 23:20:01.144337 3778 cni.go:452] [tsf-tpa/migrate-db-jzmgk 572c41526f06901c72208da7f3b96264c311c9fdd066bd107f405bff2a1ec03b network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/migrate-db-jzmgk 572c41526f06901c72208da7f3b96264c311c9fdd066bd107f405bff2a1ec03b network default NAD default NAD key ] I0425 23:20:01.238974 3778 cni.go:473] [tsf-tpa/migrate-db-jzmgk 572c41526f06901c72208da7f3b96264c311c9fdd066bd107f405bff2a1ec03b network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/migrate-db-jzmgk 572c41526f06901c72208da7f3b96264c311c9fdd066bd107f405bff2a1ec03b network default NAD default NAD key default], result "{}", err I0425 23:20:01.277647 3778 obj_retry.go:498] Detected object tsf-tpa/migrate-db-jzmgk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:01.277692 3778 pods.go:176] Deleting pod: tsf-tpa/migrate-db-jzmgk I0425 23:20:01.279108 3778 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-jzmgk, ips: 10.128.0.42 I0425 23:20:01.279145 3778 obj_retry.go:498] Detected object tsf-tpa/migrate-db-jzmgk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:01.408266 3778 obj_retry.go:492] Detected object tsf-tpa/migrate-db-jzmgk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:01.408291 3778 obj_retry.go:492] Detected object tsf-tpa/migrate-db-jzmgk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:04.162694 3778 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x bce44a378894cb6cebc0f29ea6612eb2d9178d511d84aba0f5b45f4453616bc8 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x bce44a378894cb6cebc0f29ea6612eb2d9178d511d84aba0f5b45f4453616bc8 network default NAD default NAD key ] I0425 23:20:04.252334 3778 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x bce44a378894cb6cebc0f29ea6612eb2d9178d511d84aba0f5b45f4453616bc8 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x bce44a378894cb6cebc0f29ea6612eb2d9178d511d84aba0f5b45f4453616bc8 network default NAD default NAD key default], result "{}", err I0425 23:20:04.300454 3778 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:04.300484 3778 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x I0425 23:20:04.301765 3778 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x, ips: 10.128.0.43 I0425 23:20:04.301798 3778 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:04.520316 3778 base_network_controller_pods.go:487] [default/tsf-tpa/server-59d9599cf5-gpwbv] creating logical port tsf-tpa_server-59d9599cf5-gpwbv for pod on switch ip-10-0-2-186.ec2.internal I0425 23:20:04.520409 3778 kube.go:256] Updating pod tsf-tpa/server-59d9599cf5-gpwbv I0425 23:20:04.534848 3778 pod.go:62] [tsf-tpa/server-59d9599cf5-gpwbv] pod update took 14.461559ms I0425 23:20:04.534876 3778 base_network_controller_pods.go:951] [default/tsf-tpa/server-59d9599cf5-gpwbv] addLogicalPort annotation time took 14.497424ms I0425 23:20:04.536062 3778 pods.go:271] [tsf-tpa/server-59d9599cf5-gpwbv] addLogicalPort took 15.761596ms, libovsdb time 795.464µs I0425 23:20:05.170859 3778 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:05.170885 3778 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619320-rth5x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:06.020576 3778 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-ckztl] creating logical port tsf-tpa_importer-86bcdf477b-ckztl for pod on switch ip-10-0-2-186.ec2.internal I0425 23:20:06.020686 3778 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-ckztl I0425 23:20:06.035221 3778 pod.go:62] [tsf-tpa/importer-86bcdf477b-ckztl] pod update took 14.563218ms I0425 23:20:06.035251 3778 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-ckztl] addLogicalPort annotation time took 14.598124ms I0425 23:20:06.036722 3778 pods.go:271] [tsf-tpa/importer-86bcdf477b-ckztl] addLogicalPort took 16.169767ms, libovsdb time 919.663µs I0425 23:20:06.256528 3778 obj_retry.go:498] Detected object tsf-tpa/create-importers-6x2tb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:06.257136 3778 obj_retry.go:498] Detected object tsf-tpa/create-importers-6x2tb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:07.277068 3778 obj_retry.go:492] Detected object tsf-tpa/create-importers-6x2tb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:07.277110 3778 obj_retry.go:492] Detected object tsf-tpa/create-importers-6x2tb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0425 23:20:08.163496 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0425 23:20:09.382544 3778 cni.go:452] [tsf-tpa/server-59d9599cf5-gpwbv 76f3c0ad4ac74fa57f917e1a8d3dd2089585653ff3914da5c6d8ac005f39c222 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-59d9599cf5-gpwbv 76f3c0ad4ac74fa57f917e1a8d3dd2089585653ff3914da5c6d8ac005f39c222 network default NAD default NAD key ] I0425 23:20:09.392178 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-59d9599cf5-gpwbv, hostIfaceName: 76f3c0ad4ac74fa, network: default, NAD default, SandboxID: "76f3c0ad4ac74fa57f917e1a8d3dd2089585653ff3914da5c6d8ac005f39c222", PCI device ID: , UID: "3d071dd0-a5bb-48e5-976d-02c4a64af190", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0425 23:20:09.452057 3778 cni.go:473] [tsf-tpa/server-59d9599cf5-gpwbv 76f3c0ad4ac74fa57f917e1a8d3dd2089585653ff3914da5c6d8ac005f39c222 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-59d9599cf5-gpwbv 76f3c0ad4ac74fa57f917e1a8d3dd2089585653ff3914da5c6d8ac005f39c222 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:ea:8d:b2:50:bd\",\"name\":\"76f3c0ad4ac74fa\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c021316c-8b59-4320-95bc-772b5d252736\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 23:20:09.983934 3778 cni.go:452] [tsf-tpa/importer-86bcdf477b-ckztl dfbdfd5d25f6bbd11fae48629add278f11fd98751760f79b843e9b95cf516754 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-ckztl dfbdfd5d25f6bbd11fae48629add278f11fd98751760f79b843e9b95cf516754 network default NAD default NAD key ] I0425 23:20:09.995304 3778 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-ckztl, hostIfaceName: dfbdfd5d25f6bbd, network: default, NAD default, SandboxID: "dfbdfd5d25f6bbd11fae48629add278f11fd98751760f79b843e9b95cf516754", PCI device ID: , UID: "6442d78d-6d05-4bc2-ae10-32611b48aff1", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0425 23:20:10.053071 3778 cni.go:473] [tsf-tpa/importer-86bcdf477b-ckztl dfbdfd5d25f6bbd11fae48629add278f11fd98751760f79b843e9b95cf516754 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-ckztl dfbdfd5d25f6bbd11fae48629add278f11fd98751760f79b843e9b95cf516754 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:20:03:9b:98:5f\",\"name\":\"dfbdfd5d25f6bbd\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c62b1637-136d-4f03-8d29-0081d5adf0c7\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 23:20:10.195695 3778 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0425 23:20:11.220010 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server W0425 23:20:12.149572 3778 util.go:959] Failed to build node endpoints for node ip-10-0-2-186.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0425 23:20:14.168363 3778 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:14.168986 3778 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:14.190241 3778 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:14.190265 3778 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:14.192785 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0425 23:20:14.192813 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0425 23:20:14.886936 3778 namespace.go:98] [default-tenant] adding namespace I0425 23:20:14.887829 3778 namespace.go:102] [default-tenant] adding namespace took 868.245µs I0425 23:20:14.890802 3778 namespace.go:98] [integration-service] adding namespace I0425 23:20:14.890856 3778 namespace.go:98] [build-service] adding namespace I0425 23:20:14.890911 3778 namespace.go:98] [image-controller] adding namespace I0425 23:20:14.891523 3778 namespace.go:102] [integration-service] adding namespace took 705.746µs I0425 23:20:14.892149 3778 namespace.go:102] [build-service] adding namespace took 1.271669ms I0425 23:20:14.892631 3778 namespace.go:102] [image-controller] adding namespace took 1.705414ms I0425 23:20:14.920809 3778 namespace.go:142] [default-tenant] updating namespace I0425 23:20:14.941696 3778 namespace.go:142] [default-tenant] updating namespace I0425 23:20:14.950226 3778 namespace.go:142] [integration-service] updating namespace I0425 23:20:14.973776 3778 namespace.go:142] [integration-service] updating namespace I0425 23:20:14.978635 3778 namespace.go:142] [build-service] updating namespace I0425 23:20:15.005856 3778 namespace.go:142] [image-controller] updating namespace I0425 23:20:15.007809 3778 namespace.go:142] [build-service] updating namespace I0425 23:20:15.023687 3778 namespace.go:142] [image-controller] updating namespace I0425 23:20:15.116163 3778 namespace.go:142] [build-service] updating namespace I0425 23:20:15.129169 3778 namespace.go:142] [integration-service] updating namespace I0425 23:20:15.155810 3778 namespace.go:98] [release-service] adding namespace I0425 23:20:15.156845 3778 namespace.go:102] [release-service] adding namespace took 1.006214ms I0425 23:20:15.185715 3778 namespace.go:142] [release-service] updating namespace I0425 23:20:15.201544 3778 namespace.go:142] [release-service] updating namespace I0425 23:20:15.275738 3778 namespace.go:142] [konflux-ui] updating namespace I0425 23:20:15.327866 3778 namespace.go:98] [konflux-info] adding namespace I0425 23:20:15.328727 3778 namespace.go:102] [konflux-info] adding namespace took 835.54µs I0425 23:20:15.412932 3778 namespace.go:98] [namespace-lister] adding namespace I0425 23:20:15.413793 3778 namespace.go:102] [namespace-lister] adding namespace took 838.786µs I0425 23:20:15.467604 3778 namespace.go:98] [enterprise-contract-service] adding namespace I0425 23:20:15.467801 3778 namespace.go:142] [image-controller] updating namespace I0425 23:20:15.468642 3778 namespace.go:102] [enterprise-contract-service] adding namespace took 1.015271ms I0425 23:20:15.554713 3778 namespace.go:142] [konflux-info] updating namespace I0425 23:20:15.693871 3778 namespace.go:142] [default-tenant] updating namespace I0425 23:20:15.730724 3778 namespace.go:98] [segment-bridge] adding namespace I0425 23:20:15.731684 3778 namespace.go:102] [segment-bridge] adding namespace took 935.614µs I0425 23:20:15.755947 3778 namespace.go:142] [konflux-info] updating namespace I0425 23:20:15.817563 3778 namespace.go:142] [namespace-lister] updating namespace I0425 23:20:15.869842 3778 namespace.go:98] [konflux-cli] adding namespace I0425 23:20:15.870717 3778 namespace.go:102] [konflux-cli] adding namespace took 852.606µs I0425 23:20:15.963898 3778 namespace.go:142] [namespace-lister] updating namespace I0425 23:20:16.091629 3778 namespace.go:142] [enterprise-contract-service] updating namespace I0425 23:20:16.169721 3778 namespace.go:142] [enterprise-contract-service] updating namespace I0425 23:20:16.190973 3778 namespace.go:142] [segment-bridge] updating namespace I0425 23:20:16.241860 3778 namespace.go:142] [segment-bridge] updating namespace I0425 23:20:16.245578 3778 namespace.go:142] [konflux-cli] updating namespace I0425 23:20:16.315768 3778 namespace.go:142] [konflux-cli] updating namespace I0425 23:20:16.450571 3778 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0425 23:20:16.451212 3778 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0425 23:20:16.451230 3778 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0425 23:20:16.453901 3778 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0425 23:20:16.765537 3778 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0425 23:20:16.765568 3778 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0425 23:20:16.766550 3778 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0425 23:20:18.067215 3778 namespace.go:142] [build-service] updating namespace I0425 23:20:18.371349 3778 namespace.go:142] [default-tenant] updating namespace I0425 23:20:18.549665 3778 namespace.go:142] [enterprise-contract-service] updating namespace I0425 23:20:18.751245 3778 namespace.go:142] [image-controller] updating namespace I0425 23:20:18.955864 3778 namespace.go:142] [integration-service] updating namespace I0425 23:20:19.174835 3778 namespace.go:142] [konflux-cli] updating namespace I0425 23:20:19.425487 3778 namespace.go:142] [konflux-info] updating namespace I0425 23:20:19.703760 3778 namespace.go:142] [namespace-lister] updating namespace W0425 23:20:19.823607 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0425 23:20:19.944765 3778 namespace.go:142] [release-service] updating namespace I0425 23:20:20.153987 3778 namespace.go:142] [segment-bridge] updating namespace I0425 23:20:20.326832 3778 namespace.go:142] [build-service] updating namespace I0425 23:20:20.364482 3778 namespace.go:142] [default-tenant] updating namespace I0425 23:20:20.396791 3778 namespace.go:142] [enterprise-contract-service] updating namespace I0425 23:20:20.433945 3778 namespace.go:142] [image-controller] updating namespace I0425 23:20:20.477926 3778 namespace.go:142] [integration-service] updating namespace I0425 23:20:20.520907 3778 namespace.go:142] [konflux-cli] updating namespace I0425 23:20:20.566661 3778 namespace.go:142] [konflux-info] updating namespace I0425 23:20:20.646666 3778 namespace.go:142] [namespace-lister] updating namespace I0425 23:20:20.648860 3778 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=85 I0425 23:20:20.690579 3778 namespace.go:142] [release-service] updating namespace I0425 23:20:20.722709 3778 namespace.go:142] [segment-bridge] updating namespace I0425 23:20:20.822675 3778 namespace.go:142] [build-service] updating namespace W0425 23:20:20.825713 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0425 23:20:20.904671 3778 namespace.go:142] [default-tenant] updating namespace I0425 23:20:20.981172 3778 namespace.go:142] [enterprise-contract-service] updating namespace I0425 23:20:21.028130 3778 namespace.go:142] [image-controller] updating namespace I0425 23:20:21.085688 3778 namespace.go:142] [integration-service] updating namespace I0425 23:20:21.155580 3778 namespace.go:142] [konflux-cli] updating namespace I0425 23:20:21.258842 3778 namespace.go:142] [konflux-info] updating namespace I0425 23:20:21.361071 3778 namespace.go:142] [namespace-lister] updating namespace I0425 23:20:21.450874 3778 namespace.go:142] [release-service] updating namespace I0425 23:20:21.555780 3778 namespace.go:142] [segment-bridge] updating namespace W0425 23:20:22.843169 3778 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0425 23:20:27.902940 3778 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0425 23:20:27.903622 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0425 23:20:28.113766 3778 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0425 23:20:29.121294 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0425 23:20:29.129470 3778 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0425 23:20:29.908998 3778 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0425 23:20:29.909026 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0425 23:20:30.733359 3778 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-2zdgd-j4fkx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:30.734016 3778 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-2zdgd-j4fkx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:30.915364 3778 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-2zdgd-j4fkx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:30.915397 3778 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-2zdgd-j4fkx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0425 23:20:32.160238 3778 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0425 23:20:32.160256 3778 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0425 23:20:32.936732 3778 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0425 23:20:32.936753 3778 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy I0425 23:20:39.214195 3778 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-4pppj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:39.214866 3778 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-4pppj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:20:40.193180 3778 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-4pppj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:40.193203 3778 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-4pppj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:20:40.200234 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-4pppj I0425 23:20:40.200263 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-4pppj W0425 23:20:45.017840 3778 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0425 23:21:08.907671 3778 namespace.go:142] [openshift-pipelines] updating namespace I0425 23:21:38.358922 3778 obj_retry.go:498] Detected object konflux-ui/proxy-5c49569bfb-jld8z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:21:38.359592 3778 obj_retry.go:498] Detected object konflux-ui/proxy-5c49569bfb-jld8z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:21:39.344015 3778 obj_retry.go:492] Detected object konflux-ui/proxy-5c49569bfb-jld8z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:21:39.344037 3778 obj_retry.go:492] Detected object konflux-ui/proxy-5c49569bfb-jld8z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:21:39.350610 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-5c49569bfb-jld8z I0425 23:21:39.350636 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-5c49569bfb-jld8z I0425 23:21:40.762520 3778 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0425 23:21:44.682042 3778 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 23:21:44.682600 3778 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 23:21:44.718866 3778 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:21:44.718888 3778 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 23:21:44.723525 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0425 23:21:44.723549 3778 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0425 23:22:11.329454 3778 namespace.go:98] [default-managed-tenant-se4u3] adding namespace I0425 23:22:11.330293 3778 namespace.go:102] [default-managed-tenant-se4u3] adding namespace took 817.693µs I0425 23:22:11.404868 3778 namespace.go:142] [default-managed-tenant-se4u3] updating namespace I0425 23:22:11.422714 3778 namespace.go:142] [default-managed-tenant-se4u3] updating namespace I0425 23:22:12.416638 3778 namespace.go:142] [default-managed-tenant-se4u3] updating namespace I0425 23:22:12.550565 3778 namespace.go:142] [default-managed-tenant-se4u3] updating namespace I0425 23:22:12.645779 3778 namespace.go:142] [default-managed-tenant-se4u3] updating namespace I0425 23:22:18.747726 3778 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8