+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-104.ec2.internal ++ [[ -n ip-10-0-0-104.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-104.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0424 02:56:58.333949150 - disable conntrack on geneve port' I0424 02:56:58.333949150 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0424 02:56:58.361405945 - starting ovnkube-node + echo 'I0424 02:56:58.361405945 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-104.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-104.ec2.internal --init-node ip-10-0-0-104.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-104.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0424 02:56:58.402867 3689 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0424 02:56:58.402917 3689 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-92e50c4391.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0424 02:56:58.404391 3689 kube.go:431] Waiting for certificate I0424 02:56:58.404414 3689 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0424 02:56:58.404446 3689 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0424 02:56:58.405441 3689 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0424 02:56:58.449313 3689 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0424 02:56:58.449331 3689 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0424 02:56:58.455162 3689 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0424 02:56:58.455251 3689 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-57k78" I0424 02:56:58.461307 3689 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-57k78" I0424 02:56:58.461365 3689 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0424 02:56:59.404620 3689 kube.go:438] Certificate found I0424 02:56:59.405669 3689 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0424 02:56:59.406473 3689 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0424 02:56:59.466818 3689 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-25 02:51:58 +0000 UTC" deadline="2026-04-24 21:05:39.540174867 +0000 UTC" I0424 02:56:59.466856 3689 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h8m40.073329603s" I0424 02:56:59.470282 3689 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0424 02:56:59.490827 3689 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0424 02:56:59.490896 3689 server.go:101] MetricServer registers OVS metrics I0424 02:56:59.491204 3689 server.go:105] MetricServer registers OVN DB metrics I0424 02:56:59.493940 3689 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0424 02:56:59.494006 3689 factory.go:561] Starting watch factory I0424 02:56:59.494109 3689 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494119 3689 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494124 3689 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494129 3689 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494133 3689 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494143 3689 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494144 3689 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494153 3689 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494483 3689 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494630 3689 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494703 3689 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.494834 3689 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.496526 3689 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0424 02:56:59.496555 3689 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0424 02:56:59.497024 3689 controller_manager.go:370] Starting the ovnkube controller I0424 02:56:59.497037 3689 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-104.ec2.internal I0424 02:56:59.497089 3689 controller_manager.go:395] NBDB zone sync took: 44.481µs I0424 02:56:59.497101 3689 factory.go:561] Starting watch factory I0424 02:56:59.509465 3689 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0424 02:56:59.509501 3689 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0424 02:56:59.510097 3689 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.510810 3689 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.511977 3689 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.512948 3689 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.513121 3689 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.513478 3689 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0424 02:56:59.513533 3689 server.go:109] MetricServer registers OVN Controller metrics I0424 02:56:59.518401 3689 server.go:113] MetricServer registers OVN Northd metrics I0424 02:56:59.521898 3689 metrics.go:519] OVN Metrics Server starts to run ... I0424 02:56:59.531859 3689 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0424 02:56:59.600351 3689 factory.go:1977] *v1.Node informer cache synced successfully I0424 02:56:59.600368 3689 factory.go:1977] *v1.Service informer cache synced successfully I0424 02:56:59.600373 3689 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0424 02:56:59.600377 3689 factory.go:1977] *v1.Pod informer cache synced successfully I0424 02:56:59.600381 3689 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0424 02:56:59.600386 3689 factory.go:1977] *v1.Namespace informer cache synced successfully I0424 02:56:59.600460 3689 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 02:56:59.600470 3689 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 02:56:59.600486 3689 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 02:56:59.600474 3689 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 02:56:59.603331 3689 factory.go:1977] *v1.Namespace informer cache synced successfully I0424 02:56:59.603343 3689 factory.go:1977] *v1.Node informer cache synced successfully I0424 02:56:59.603346 3689 factory.go:1977] *v1.Service informer cache synced successfully I0424 02:56:59.603349 3689 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0424 02:56:59.603352 3689 factory.go:1977] *v1.Pod informer cache synced successfully I0424 02:56:59.603355 3689 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0424 02:56:59.607299 3689 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 02:56:59.607314 3689 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0424 02:56:59.609038 3689 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0424 02:56:59.609049 3689 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0424 02:56:59.609095 3689 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.609109 3689 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.616047 3689 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.706078 3689 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0424 02:56:59.706091 3689 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0424 02:56:59.711226 3689 factory.go:1977] *v1.EgressIP informer cache synced successfully I0424 02:56:59.711309 3689 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.711329 3689 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.713803 3689 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.715803 3689 factory.go:1977] *v1.EgressIP informer cache synced successfully I0424 02:56:59.717226 3689 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0424 02:56:59.717292 3689 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.717309 3689 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.719469 3689 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.721522 3689 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0424 02:56:59.722820 3689 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0424 02:56:59.722877 3689 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.722889 3689 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.725177 3689 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.727187 3689 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0424 02:56:59.728271 3689 factory.go:1977] *v1.EgressService informer cache synced successfully I0424 02:56:59.728326 3689 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.728338 3689 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.730559 3689 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.732234 3689 factory.go:1977] *v1.EgressService informer cache synced successfully I0424 02:56:59.733378 3689 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0424 02:56:59.733534 3689 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0424 02:56:59.733549 3689 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0424 02:56:59.735830 3689 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0424 02:56:59.737672 3689 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0424 02:56:59.738803 3689 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0424 02:56:59.738924 3689 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.738937 3689 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.738945 3689 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.738976 3689 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.741270 3689 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.741300 3689 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0424 02:56:59.743042 3689 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0424 02:56:59.744326 3689 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0424 02:56:59.744335 3689 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0424 02:56:59.744342 3689 factory.go:680] Watch Factory start up complete, took: 250.342083ms I0424 02:56:59.744402 3689 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0424 02:56:59.748940 3689 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0424 02:56:59.748950 3689 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0424 02:56:59.748957 3689 factory.go:680] Watch Factory start up complete, took: 251.856135ms I0424 02:56:59.748964 3689 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-0-104.ec2.internal" zone I0424 02:56:59.800698 3689 default_node_network_controller.go:710] Initializing the default node network controller I0424 02:56:59.856598 3689 udn_isolation.go:101] Starting UDN host isolation manager I0424 02:56:59.861959 3689 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0424 02:56:59.913980 3689 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0424 02:56:59.914056 3689 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0424 02:56:59.914069 3689 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0424 02:56:59.943092 3689 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0424 02:56:59.943145 3689 default_node_network_controller.go:809] Node ip-10-0-0-104.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0424 02:56:59.947686 3689 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.104"] k8s.ovn.org/zone-name:ip-10-0-0-104.ec2.internal] on node ip-10-0-0-104.ec2.internal I0424 02:56:59.967062 3689 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0424 02:56:59.972591 3689 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0424 02:56:59.972733 3689 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0424 02:56:59.972864 3689 gateway_init.go:263] Preparing Gateway I0424 02:56:59.972877 3689 gateway_shared_intf.go:1687] Creating new gateway I0424 02:56:59.999051 3689 controller_manager.go:429] Waiting for node in zone sync took: 250.076617ms I0424 02:57:00.004657 3689 controller_manager.go:611] Switching to transit router for layer2 networks I0424 02:57:00.004670 3689 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-104.ec2.internal I0424 02:57:00.018916 3689 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0424 02:57:00.024990 3689 controller_manager.go:290] SCTP support detected in OVN I0424 02:57:00.025467 3689 services_controller.go:69] Creating services controller for network=default I0424 02:57:00.025533 3689 udn_enabled_svc.go:80] Waiting for service informer to sync I0424 02:57:00.025602 3689 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0424 02:57:00.025631 3689 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0424 02:57:00.025643 3689 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0424 02:57:00.025651 3689 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0424 02:57:00.025799 3689 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0424 02:57:00.025842 3689 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0424 02:57:00.025872 3689 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 47.848µs I0424 02:57:00.025879 3689 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 30.962µs I0424 02:57:00.025888 3689 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 46.685µs I0424 02:57:00.025897 3689 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0424 02:57:00.025910 3689 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0424 02:57:00.025923 3689 default_network_controller.go:325] Starting the default network controller I0424 02:57:00.025941 3689 udn_enabled_svc.go:95] Performing full resync I0424 02:57:00.025990 3689 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 11.634µs I0424 02:57:00.026183 3689 udn_enabled_svc.go:99] Waiting for handler to sync I0424 02:57:00.026193 3689 udn_enabled_svc.go:104] Starting worker I0424 02:57:00.034557 3689 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0424 02:57:00.034612 3689 repair.go:33] Syncing exgw routes took 42.927µs I0424 02:57:00.034632 3689 default_network_controller.go:414] Starting all the Watchers... I0424 02:57:00.034759 3689 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0424 02:57:00.034765 3689 namespace.go:98] [openshift-backplane] adding namespace I0424 02:57:00.034765 3689 namespace.go:98] [openshift-apiserver-operator] adding namespace I0424 02:57:00.034773 3689 namespace.go:98] [openshift-logging] adding namespace I0424 02:57:00.034785 3689 namespace.go:98] [openshift-backplane-cse] adding namespace I0424 02:57:00.034793 3689 namespace.go:98] [open-cluster-management-2ps8kd9n54vkadsmhv5dl8r11de9333h] adding namespace I0424 02:57:00.034799 3689 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0424 02:57:00.034806 3689 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0424 02:57:00.034808 3689 namespace.go:98] [openshift-backplane-srep] adding namespace I0424 02:57:00.034810 3689 namespace.go:98] [openshift-operators-redhat] adding namespace I0424 02:57:00.034805 3689 namespace.go:98] [openshift-kube-apiserver] adding namespace I0424 02:57:00.034811 3689 namespace.go:98] [kube-system] adding namespace I0424 02:57:00.034815 3689 namespace.go:98] [openshift] adding namespace I0424 02:57:00.034837 3689 namespace.go:98] [openshift-console] adding namespace I0424 02:57:00.034759 3689 namespace.go:98] [default] adding namespace I0424 02:57:00.035330 3689 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 559.388µs I0424 02:57:00.035344 3689 namespace.go:98] [openshift-service-ca] adding namespace I0424 02:57:00.035846 3689 namespace.go:102] [openshift-service-ca] adding namespace took 497.147µs I0424 02:57:00.035868 3689 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0424 02:57:00.036343 3689 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 470.61µs I0424 02:57:00.036355 3689 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0424 02:57:00.036817 3689 namespace.go:102] [openshift-backplane] adding namespace took 2.042867ms I0424 02:57:00.036830 3689 namespace.go:98] [openshift-config-operator] adding namespace I0424 02:57:00.037237 3689 namespace.go:102] [openshift-apiserver-operator] adding namespace took 2.458568ms I0424 02:57:00.037253 3689 namespace.go:98] [openshift-service-ca-operator] adding namespace I0424 02:57:00.037699 3689 namespace.go:102] [openshift-logging] adding namespace took 2.918578ms I0424 02:57:00.037711 3689 namespace.go:98] [openshift-route-controller-manager] adding namespace I0424 02:57:00.038136 3689 namespace.go:102] [openshift-backplane-cse] adding namespace took 3.339782ms I0424 02:57:00.038148 3689 namespace.go:98] [openshift-backplane-mobb] adding namespace I0424 02:57:00.038529 3689 namespace.go:102] [open-cluster-management-2ps8kd9n54vkadsmhv5dl8r11de9333h] adding namespace took 3.730005ms I0424 02:57:00.038543 3689 namespace.go:98] [openshift-infra] adding namespace I0424 02:57:00.038970 3689 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 4.164756ms I0424 02:57:00.038982 3689 namespace.go:98] [openshift-network-console] adding namespace I0424 02:57:00.039789 3689 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 4.976551ms I0424 02:57:00.039806 3689 namespace.go:98] [dedicated-admin] adding namespace I0424 02:57:00.040228 3689 namespace.go:102] [openshift-backplane-srep] adding namespace took 5.413833ms I0424 02:57:00.040240 3689 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0424 02:57:00.040644 3689 namespace.go:102] [openshift-operators-redhat] adding namespace took 5.827719ms I0424 02:57:00.040655 3689 namespace.go:98] [openshift-console-operator] adding namespace I0424 02:57:00.040996 3689 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.177512ms I0424 02:57:00.041009 3689 namespace.go:98] [openshift-network-diagnostics] adding namespace I0424 02:57:00.041468 3689 namespace.go:102] [kube-system] adding namespace took 6.646975ms I0424 02:57:00.041481 3689 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0424 02:57:00.041895 3689 namespace.go:102] [openshift] adding namespace took 7.070908ms I0424 02:57:00.041908 3689 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0424 02:57:00.042292 3689 namespace.go:102] [openshift-console] adding namespace took 7.446403ms I0424 02:57:00.042303 3689 namespace.go:98] [openshift-dns] adding namespace I0424 02:57:00.042672 3689 namespace.go:102] [default] adding namespace took 7.782949ms I0424 02:57:00.042686 3689 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0424 02:57:00.043065 3689 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.704967ms I0424 02:57:00.043076 3689 namespace.go:98] [openshift-authentication-operator] adding namespace I0424 02:57:00.043427 3689 namespace.go:102] [openshift-config-operator] adding namespace took 6.591651ms I0424 02:57:00.043438 3689 namespace.go:98] [openshift-monitoring] adding namespace I0424 02:57:00.043874 3689 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.616608ms I0424 02:57:00.043884 3689 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0424 02:57:00.044234 3689 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.516824ms I0424 02:57:00.044247 3689 namespace.go:98] [openshift-apiserver] adding namespace I0424 02:57:00.044598 3689 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.445068ms I0424 02:57:00.044609 3689 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0424 02:57:00.045001 3689 namespace.go:102] [openshift-infra] adding namespace took 6.452302ms I0424 02:57:00.045012 3689 namespace.go:98] [openshift-ingress-operator] adding namespace I0424 02:57:00.045402 3689 namespace.go:102] [openshift-network-console] adding namespace took 6.413789ms I0424 02:57:00.045414 3689 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0424 02:57:00.045794 3689 namespace.go:102] [dedicated-admin] adding namespace took 5.983082ms I0424 02:57:00.045806 3689 namespace.go:98] [openshift-machine-api] adding namespace I0424 02:57:00.046152 3689 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 5.906738ms I0424 02:57:00.046162 3689 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0424 02:57:00.046530 3689 namespace.go:102] [openshift-console-operator] adding namespace took 5.869924ms I0424 02:57:00.046541 3689 namespace.go:98] [openshift-marketplace] adding namespace W0424 02:57:00.046557 3689 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-z4cpl: pod openshift-network-diagnostics/network-check-target-z4cpl: no pod IPs found W0424 02:57:00.046570 3689 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-9q6zq: pod openshift-network-diagnostics/network-check-target-9q6zq: no pod IPs found I0424 02:57:00.047045 3689 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.031066ms I0424 02:57:00.047058 3689 namespace.go:98] [kube-public] adding namespace I0424 02:57:00.047455 3689 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 5.969624ms I0424 02:57:00.047463 3689 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0424 02:57:00.047904 3689 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 5.989914ms I0424 02:57:00.047918 3689 namespace.go:98] [openshift-insights] adding namespace I0424 02:57:00.048281 3689 namespace.go:102] [openshift-dns] adding namespace took 5.973804ms I0424 02:57:00.048292 3689 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0424 02:57:00.048718 3689 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.02746ms I0424 02:57:00.048729 3689 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0424 02:57:00.049063 3689 namespace.go:102] [openshift-authentication-operator] adding namespace took 5.982603ms I0424 02:57:00.049519 3689 namespace.go:102] [openshift-monitoring] adding namespace took 6.061027ms I0424 02:57:00.049535 3689 namespace.go:98] [openshift-backplane-tam] adding namespace I0424 02:57:00.049908 3689 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.018811ms I0424 02:57:00.049921 3689 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0424 02:57:00.050264 3689 namespace.go:102] [openshift-apiserver] adding namespace took 6.01292ms I0424 02:57:00.050274 3689 namespace.go:98] [openshift-etcd] adding namespace I0424 02:57:00.050611 3689 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 5.997997ms I0424 02:57:00.050620 3689 namespace.go:98] [openshift-ingress] adding namespace I0424 02:57:00.051061 3689 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.04383ms I0424 02:57:00.051075 3689 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0424 02:57:00.051447 3689 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.027613ms I0424 02:57:00.051459 3689 namespace.go:98] [openshift-image-registry] adding namespace I0424 02:57:00.051840 3689 namespace.go:102] [openshift-machine-api] adding namespace took 6.029911ms I0424 02:57:00.051851 3689 namespace.go:98] [openshift-controller-manager] adding namespace I0424 02:57:00.052229 3689 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.063429ms I0424 02:57:00.052239 3689 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0424 02:57:00.052553 3689 namespace.go:102] [openshift-marketplace] adding namespace took 6.007702ms I0424 02:57:00.052563 3689 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0424 02:57:00.052988 3689 namespace.go:102] [kube-public] adding namespace took 5.925094ms I0424 02:57:00.053001 3689 namespace.go:98] [openshift-backplane-cee] adding namespace I0424 02:57:00.053318 3689 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 5.851576ms I0424 02:57:00.053328 3689 namespace.go:98] [openshift-ingress-canary] adding namespace I0424 02:57:00.053776 3689 namespace.go:102] [openshift-insights] adding namespace took 5.854076ms I0424 02:57:00.053787 3689 namespace.go:98] [openshift-customer-monitoring] adding namespace I0424 02:57:00.054097 3689 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 5.800873ms I0424 02:57:00.054106 3689 namespace.go:98] [openshift-host-network] adding namespace I0424 02:57:00.054457 3689 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 5.722517ms I0424 02:57:00.054469 3689 namespace.go:98] [openshift-backplane-csm] adding namespace I0424 02:57:00.054911 3689 namespace.go:102] [openshift-backplane-tam] adding namespace took 5.371746ms I0424 02:57:00.054922 3689 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0424 02:57:00.055238 3689 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 5.312634ms I0424 02:57:00.055247 3689 namespace.go:98] [openshift-machine-config-operator] adding namespace I0424 02:57:00.055802 3689 namespace.go:102] [openshift-etcd] adding namespace took 5.522036ms I0424 02:57:00.055816 3689 namespace.go:98] [kube-node-lease] adding namespace I0424 02:57:00.056240 3689 namespace.go:102] [openshift-ingress] adding namespace took 5.61554ms I0424 02:57:00.056252 3689 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0424 02:57:00.056590 3689 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 5.509905ms I0424 02:57:00.056601 3689 namespace.go:98] [openshift-package-operator] adding namespace I0424 02:57:00.057070 3689 namespace.go:102] [openshift-image-registry] adding namespace took 5.605858ms I0424 02:57:00.057080 3689 namespace.go:98] [openshift-multus] adding namespace I0424 02:57:00.057488 3689 namespace.go:102] [openshift-controller-manager] adding namespace took 5.632712ms I0424 02:57:00.057497 3689 namespace.go:98] [openshift-console-user-settings] adding namespace I0424 02:57:00.058038 3689 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 5.794103ms I0424 02:57:00.058050 3689 namespace.go:98] [openshift-network-operator] adding namespace I0424 02:57:00.058384 3689 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 5.816742ms I0424 02:57:00.058393 3689 namespace.go:98] [openshift-kube-scheduler] adding namespace I0424 02:57:00.058856 3689 namespace.go:102] [openshift-backplane-cee] adding namespace took 5.85119ms I0424 02:57:00.058868 3689 namespace.go:98] [openshift-operators] adding namespace I0424 02:57:00.059209 3689 namespace.go:102] [openshift-ingress-canary] adding namespace took 5.877593ms I0424 02:57:00.059218 3689 namespace.go:98] [openshift-authentication] adding namespace I0424 02:57:00.059678 3689 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.88461ms I0424 02:57:00.059692 3689 namespace.go:98] [openshift-node] adding namespace I0424 02:57:00.060113 3689 namespace.go:102] [openshift-host-network] adding namespace took 6.002433ms I0424 02:57:00.060123 3689 namespace.go:98] [openshift-config] adding namespace I0424 02:57:00.060523 3689 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.049531ms I0424 02:57:00.060531 3689 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0424 02:57:00.060926 3689 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 5.998168ms I0424 02:57:00.060942 3689 namespace.go:98] [openshift-cluster-version] adding namespace I0424 02:57:00.061290 3689 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.039175ms I0424 02:57:00.061668 3689 namespace.go:102] [kube-node-lease] adding namespace took 5.848421ms I0424 02:57:00.062032 3689 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.775816ms W0424 02:57:00.062369 3689 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-pskqg: pod openshift-multus/network-metrics-daemon-pskqg: no pod IPs found W0424 02:57:00.062380 3689 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-rpx7j: pod openshift-multus/network-metrics-daemon-rpx7j: no pod IPs found I0424 02:57:00.062404 3689 namespace.go:102] [openshift-package-operator] adding namespace took 5.798797ms I0424 02:57:00.062771 3689 namespace.go:102] [openshift-multus] adding namespace took 5.686906ms I0424 02:57:00.062783 3689 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0424 02:57:00.063215 3689 namespace.go:102] [openshift-console-user-settings] adding namespace took 5.712935ms I0424 02:57:00.063572 3689 namespace.go:102] [openshift-network-operator] adding namespace took 5.517277ms I0424 02:57:00.063584 3689 namespace.go:98] [openshift-dns-operator] adding namespace I0424 02:57:00.063891 3689 namespace.go:102] [openshift-kube-scheduler] adding namespace took 5.494031ms I0424 02:57:00.064276 3689 namespace.go:102] [openshift-operators] adding namespace took 5.403398ms I0424 02:57:00.064287 3689 namespace.go:98] [openshift-must-gather-operator] adding namespace I0424 02:57:00.064638 3689 namespace.go:102] [openshift-authentication] adding namespace took 5.414959ms I0424 02:57:00.064997 3689 namespace.go:102] [openshift-node] adding namespace took 5.300037ms I0424 02:57:00.065006 3689 namespace.go:98] [openshift-config-managed] adding namespace I0424 02:57:00.065758 3689 namespace.go:102] [openshift-config] adding namespace took 5.630734ms I0424 02:57:00.066106 3689 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 5.570671ms I0424 02:57:00.066116 3689 namespace.go:98] [openshift-network-node-identity] adding namespace I0424 02:57:00.066491 3689 namespace.go:102] [openshift-cluster-version] adding namespace took 5.543819ms I0424 02:57:00.066867 3689 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 4.079576ms I0424 02:57:00.067213 3689 namespace.go:102] [openshift-dns-operator] adding namespace took 3.625688ms I0424 02:57:00.067653 3689 namespace.go:102] [openshift-must-gather-operator] adding namespace took 3.360688ms I0424 02:57:00.067992 3689 namespace.go:102] [openshift-config-managed] adding namespace took 2.98198ms I0424 02:57:00.068376 3689 namespace.go:102] [openshift-network-node-identity] adding namespace took 2.255018ms W0424 02:57:00.068442 3689 master.go:458] Unable to parse local node ip-10-0-0-104.ec2.internal chassis-id annotation. Chassis may be removed during sync W0424 02:57:00.068450 3689 master.go:471] Unable to parse remote node ip-10-0-1-139.ec2.internal chassis-id annotation. Chassis may be removed during sync I0424 02:57:00.068455 3689 master.go:481] Removing stale chassis with ID/Name: 0cadfb8c-e596-4e92-902b-25491e025f63, hostname: ip-10-0-0-104.ec2.internal I0424 02:57:00.068850 3689 master.go:548] Adding or Updating local node "ip-10-0-0-104.ec2.internal" for network "default" I0424 02:57:00.068918 3689 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-139.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-139.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-139.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-139.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-139.ec2.internal I0424 02:57:00.069575 3689 zone_ic_handler.go:195] Time taken to create transit switch: 336.256µs I0424 02:57:00.069590 3689 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-137.ec2.internal for the network default I0424 02:57:00.070439 3689 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-137.ec2.internal" on network "default" took: 1.209326ms I0424 02:57:00.072412 3689 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-104.ec2.internal for the network default I0424 02:57:00.072573 3689 zone_ic_handler.go:195] Time taken to create transit switch: 150.266µs I0424 02:57:00.073190 3689 default_network_controller.go:823] Node add failed for ip-10-0-0-104.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-104.ec2.internal, error getting gateway config for node ip-10-0-0-104.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-104.ec2.internal", failed to update chassis to local for local node ip-10-0-0-104.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-104.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-104.ec2.internal] E0424 02:57:00.073225 3689 obj_retry.go:575] Failed to create *v1.Node ip-10-0-0-104.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-104.ec2.internal, error getting gateway config for node ip-10-0-0-104.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-104.ec2.internal", failed to update chassis to local for local node ip-10-0-0-104.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-104.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-104.ec2.internal] I0424 02:57:00.073259 3689 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0424 02:57:00.073278 3689 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0424 02:57:00.073294 3689 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0424 02:57:00.073305 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-104.ec2.internal in network "default" I0424 02:57:00.073284 3689 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-104.ec2.internal", UID:"ce798f18-38d5-4a72-a413-7b51a6e4a171", APIVersion:"v1", ResourceVersion:"9175", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-104.ec2.internal, error getting gateway config for node ip-10-0-0-104.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-104.ec2.internal", failed to update chassis to local for local node ip-10-0-0-104.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-104.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-104.ec2.internal] I0424 02:57:00.073334 3689 node_tracker.go:235] Node ip-10-0-0-104.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-104.ec2.internal" I0424 02:57:00.073347 3689 node_tracker.go:169] Node ip-10-0-0-104.ec2.internal switch + router changed, syncing services in network "default" I0424 02:57:00.073359 3689 services_controller.go:574] Full service sync requested for network=default I0424 02:57:00.073367 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-139.ec2.internal in network "default" I0424 02:57:00.073377 3689 node_tracker.go:235] Node ip-10-0-1-139.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-139.ec2.internal" I0424 02:57:00.073383 3689 node_tracker.go:169] Node ip-10-0-1-139.ec2.internal switch + router changed, syncing services in network "default" I0424 02:57:00.073386 3689 services_controller.go:574] Full service sync requested for network=default I0424 02:57:00.073390 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-137.ec2.internal in network "default" I0424 02:57:00.073432 3689 node_tracker.go:169] Node ip-10-0-2-137.ec2.internal switch + router changed, syncing services in network "default" I0424 02:57:00.073439 3689 services_controller.go:574] Full service sync requested for network=default I0424 02:57:00.086629 3689 iptables.go:108] Creating table: filter chain: FORWARD I0424 02:57:00.092429 3689 iptables.go:108] Creating table: filter chain: OUTPUT I0424 02:57:00.098349 3689 gateway_shared_intf.go:1816] Gateway Creation Complete I0424 02:57:00.098362 3689 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-104.ec2.internal","mac-address":"12:01:dc:18:73:a1","ip-addresses":["10.0.0.104/24"],"ip-address":"10.0.0.104/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:0cadfb8c-e596-4e92-902b-25491e025f63 k8s.ovn.org/node-encap-ips:["10.0.0.104"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.104/24"} k8s.ovn.org/zone-name:ip-10-0-0-104.ec2.internal] on node ip-10-0-0-104.ec2.internal I0424 02:57:00.115219 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-104.ec2.internal in network "default" I0424 02:57:00.115266 3689 node_tracker.go:169] Node ip-10-0-0-104.ec2.internal switch + router changed, syncing services in network "default" I0424 02:57:00.115268 3689 master.go:548] Adding or Updating local node "ip-10-0-0-104.ec2.internal" for network "default" I0424 02:57:00.115280 3689 services_controller.go:574] Full service sync requested for network=default I0424 02:57:00.118945 3689 default_node_network_controller.go:519] Node connection status = connected W0424 02:57:00.120105 3689 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-rpx7j: pod openshift-multus/network-metrics-daemon-rpx7j: no pod IPs found W0424 02:57:00.120127 3689 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-9q6zq: pod openshift-network-diagnostics/network-check-target-9q6zq: no pod IPs found I0424 02:57:00.120993 3689 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-104.ec2.internal for the network default I0424 02:57:00.121176 3689 zone_ic_handler.go:195] Time taken to create transit switch: 171.63µs I0424 02:57:00.174104 3689 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0424 02:57:00.174121 3689 services_controller.go:208] Setting up event handlers for services for network=default I0424 02:57:00.174162 3689 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0424 02:57:00.174194 3689 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0424 02:57:00.174205 3689 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0424 02:57:00.274778 3689 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0424 02:57:00.274793 3689 repair.go:60] Starting repairing loop for services I0424 02:57:00.274864 3689 repair.go:131] Deleted 0 stale service LBs I0424 02:57:00.274880 3689 repair.go:137] Deleted 0 stale Chassis Template Vars I0424 02:57:00.274894 3689 repair.go:62] Finished repairing loop for services: 102.436µs I0424 02:57:00.274920 3689 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0424 02:57:00.274929 3689 services_controller.go:254] Starting workers for network=default I0424 02:57:00.275927 3689 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-9q6zq] creating logical port openshift-network-diagnostics_network-check-target-9q6zq for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:00.276129 3689 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-9q6zq I0424 02:57:00.276142 3689 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-rpx7j] creating logical port openshift-multus_network-metrics-daemon-rpx7j for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:00.276250 3689 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-pskqg, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-pskqg: suppressed error logged: pod openshift-multus/network-metrics-daemon-pskqg: no pod IPs found I0424 02:57:00.276309 3689 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-rpx7j I0424 02:57:00.276370 3689 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-z4cpl, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-z4cpl: suppressed error logged: pod openshift-network-diagnostics/network-check-target-z4cpl: no pod IPs found W0424 02:57:00.278988 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0424 02:57:00.279185 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0424 02:57:00.290815 3689 pod.go:62] [openshift-network-diagnostics/network-check-target-9q6zq] pod update took 14.747014ms I0424 02:57:00.290835 3689 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-9q6zq] addLogicalPort annotation time took 14.771177ms I0424 02:57:00.291703 3689 pods.go:271] [openshift-network-diagnostics/network-check-target-9q6zq] addLogicalPort took 15.787244ms, libovsdb time 659.784µs I0424 02:57:00.295372 3689 pod.go:62] [openshift-multus/network-metrics-daemon-rpx7j] pod update took 19.091259ms I0424 02:57:00.295391 3689 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-rpx7j] addLogicalPort annotation time took 19.112795ms I0424 02:57:00.296242 3689 pods.go:271] [openshift-multus/network-metrics-daemon-rpx7j] addLogicalPort took 20.10952ms, libovsdb time 624.941µs I0424 02:57:00.296792 3689 admin_network_policy_controller.go:227] Starting controller default-network-controller I0424 02:57:00.296814 3689 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0424 02:57:00.296827 3689 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0424 02:57:00.296833 3689 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0424 02:57:00.297080 3689 repair.go:29] Repairing admin network policies took 240.263µs I0424 02:57:00.297250 3689 repair.go:92] Repairing baseline admin network policies took 157.467µs I0424 02:57:00.297677 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0424 02:57:00.297692 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0424 02:57:00.299025 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0424 02:57:00.299045 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0424 02:57:00.299054 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0424 02:57:00.300607 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0424 02:57:00.300624 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0424 02:57:00.300632 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0424 02:57:00.301203 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0424 02:57:00.301226 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0424 02:57:00.301232 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0424 02:57:00.302592 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0424 02:57:00.302609 3689 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0424 02:57:00.302616 3689 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0424 02:57:00.304264 3689 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0424 02:57:00.304291 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0424 02:57:00.304301 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0424 02:57:00.306048 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0424 02:57:00.306078 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0424 02:57:00.306090 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0424 02:57:00.306663 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0424 02:57:00.306684 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0424 02:57:00.306697 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0424 02:57:00.307763 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0424 02:57:00.307782 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0424 02:57:00.307791 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0424 02:57:00.308361 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0424 02:57:00.308379 3689 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0424 02:57:00.308387 3689 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0424 02:57:00.308988 3689 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0424 02:57:00.309005 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0424 02:57:00.309015 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0424 02:57:00.310631 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0424 02:57:00.310650 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0424 02:57:00.311415 3689 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0424 02:57:00.311433 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0424 02:57:00.312091 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0424 02:57:00.312107 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0424 02:57:00.312115 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0424 02:57:00.312544 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0424 02:57:00.312562 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0424 02:57:00.312571 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0424 02:57:00.313523 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0424 02:57:00.313547 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0424 02:57:00.313581 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0424 02:57:00.314244 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0424 02:57:00.314261 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0424 02:57:00.314267 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0424 02:57:00.315432 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0424 02:57:00.315455 3689 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0424 02:57:00.315465 3689 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0424 02:57:00.316434 3689 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0424 02:57:00.316454 3689 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0424 02:57:00.316466 3689 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0424 02:57:00.318144 3689 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0424 02:57:00.318161 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0424 02:57:00.318170 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0424 02:57:00.318585 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0424 02:57:00.318602 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0424 02:57:00.318610 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0424 02:57:00.319044 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0424 02:57:00.319060 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0424 02:57:00.319068 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0424 02:57:00.319880 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0424 02:57:00.319896 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0424 02:57:00.319903 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0424 02:57:00.320403 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0424 02:57:00.320416 3689 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0424 02:57:00.320424 3689 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0424 02:57:00.321319 3689 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0424 02:57:00.321338 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0424 02:57:00.321345 3689 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0424 02:57:00.321683 3689 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0424 02:57:00.321701 3689 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0424 02:57:00.321708 3689 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0424 02:57:00.322193 3689 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0424 02:57:00.322209 3689 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0424 02:57:00.322214 3689 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0424 02:57:00.322445 3689 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0424 02:57:00.322458 3689 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0424 02:57:00.322466 3689 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0424 02:57:00.323384 3689 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0424 02:57:00.323403 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0424 02:57:00.323414 3689 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0424 02:57:00.324546 3689 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0424 02:57:00.324563 3689 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0424 02:57:00.324573 3689 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0424 02:57:00.326103 3689 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0424 02:57:00.331705 3689 egressfirewall.go:374] Starting EgressFirewall controller I0424 02:57:00.331734 3689 controller.go:133] Adding controller egress-firewall-controller event handlers I0424 02:57:00.331738 3689 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0424 02:57:00.331756 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0424 02:57:00.331769 3689 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0424 02:57:00.331759 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0424 02:57:00.331777 3689 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0424 02:57:00.332008 3689 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0424 02:57:00.332020 3689 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0424 02:57:00.332025 3689 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0424 02:57:00.332034 3689 egressqos.go:193] Setting up event handlers for EgressQoS I0424 02:57:00.332045 3689 egressfirewall.go:723] Syncing node "ip-10-0-0-104.ec2.internal" for egress firewall I0424 02:57:00.332055 3689 egressfirewall.go:723] Syncing node "ip-10-0-1-139.ec2.internal" for egress firewall I0424 02:57:00.332059 3689 egressfirewall.go:723] Syncing node "ip-10-0-2-137.ec2.internal" for egress firewall I0424 02:57:00.332145 3689 egressqos.go:245] Starting EgressQoS Controller I0424 02:57:00.332154 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0424 02:57:00.332160 3689 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0424 02:57:00.332165 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0424 02:57:00.332169 3689 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0424 02:57:00.332174 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0424 02:57:00.332178 3689 shared_informer.go:356] "Caches are synced" controller="egressqos" I0424 02:57:00.332182 3689 egressqos.go:259] Repairing EgressQoSes I0424 02:57:00.332186 3689 egressqos.go:399] Starting repairing loop for egressqos I0424 02:57:00.332255 3689 egressqos.go:401] Finished repairing loop for egressqos: 69.471µs I0424 02:57:00.332271 3689 egressservice_zone.go:132] Setting up event handlers for Egress Services I0424 02:57:00.332304 3689 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-104.ec2.internal I0424 02:57:00.332335 3689 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-104.ec2.internal : 32.391µs I0424 02:57:00.332519 3689 egressservice_zone.go:210] Starting Egress Services Controller I0424 02:57:00.332532 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0424 02:57:00.332540 3689 shared_informer.go:356] "Caches are synced" controller="egressservices" I0424 02:57:00.332545 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0424 02:57:00.332550 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0424 02:57:00.332555 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0424 02:57:00.332559 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0424 02:57:00.332564 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0424 02:57:00.332568 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0424 02:57:00.332572 3689 egressservice_zone.go:228] Repairing Egress Services I0424 02:57:00.332857 3689 master_controller.go:88] Starting Admin Policy Based Route Controller I0424 02:57:00.332869 3689 external_controller.go:278] Starting Admin Policy Based Route Controller I0424 02:57:00.332879 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-139.ec2.internal I0424 02:57:00.332893 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-139.ec2.internal: 12.808µs I0424 02:57:00.332910 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-137.ec2.internal I0424 02:57:00.332926 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-137.ec2.internal: 21.401µs I0424 02:57:00.332936 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-104.ec2.internal I0424 02:57:00.332940 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-104.ec2.internal: 5.984µs I0424 02:57:00.333102 3689 default_network_controller.go:585] Completing all the Watchers took 298.460764ms I0424 02:57:00.333117 3689 default_network_controller.go:589] Starting unidling controllers I0424 02:57:00.333129 3689 unidle.go:46] Registering OVN SB ControllerEvent handler I0424 02:57:00.333136 3689 unidle.go:63] Populating Initial ContollerEvent events I0424 02:57:00.333145 3689 unidle.go:79] Setting up event handlers for services I0424 02:57:00.333192 3689 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0424 02:57:00.620205 3689 default_node_network_controller.go:519] Node connection status = connected I0424 02:57:01.059337 3689 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0424 02:57:01.119460 3689 default_node_network_controller.go:519] Node connection status = connected I0424 02:57:01.135393 3689 bridgeconfig.go:537] Gateway is ready I0424 02:57:01.135408 3689 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0424 02:57:01.146523 3689 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.104. Updating OVN encap IP. I0424 02:57:01.151939 3689 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.104 - it is already configured I0424 02:57:01.151953 3689 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.104/24:{} 172.20.0.1/32:{}]. Updating annotations. I0424 02:57:01.152164 3689 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.104/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-104.ec2.internal","mac-address":"12:01:dc:18:73:a1","ip-addresses":["10.0.0.104/24"],"ip-address":"10.0.0.104/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:0cadfb8c-e596-4e92-902b-25491e025f63 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.104/24"}] on node ip-10-0-0-104.ec2.internal I0424 02:57:01.166749 3689 master.go:548] Adding or Updating local node "ip-10-0-0-104.ec2.internal" for network "default" I0424 02:57:01.166755 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-104.ec2.internal in network "default" I0424 02:57:01.166891 3689 egressfirewall.go:723] Syncing node "ip-10-0-0-104.ec2.internal" for egress firewall I0424 02:57:01.166938 3689 node_tracker.go:169] Node ip-10-0-0-104.ec2.internal switch + router changed, syncing services in network "default" I0424 02:57:01.166960 3689 services_controller.go:574] Full service sync requested for network=default I0424 02:57:01.166787 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-104.ec2.internal I0424 02:57:01.166983 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-104.ec2.internal: 197.028µs W0424 02:57:01.167611 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0424 02:57:01.167650 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0424 02:57:01.168299 3689 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.0.104 I0424 02:57:01.168417 3689 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-104.ec2.internal I0424 02:57:01.172022 3689 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-104.ec2.internal. Updating no re-route policies I0424 02:57:01.172106 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-x2m8g I0424 02:57:01.172125 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5xngg I0424 02:57:01.172135 3689 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-dk8km I0424 02:57:01.172146 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5xngg I0424 02:57:01.172146 3689 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-104.ec2.internal I0424 02:57:01.172159 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-68fgb I0424 02:57:01.172163 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-zpz7f I0424 02:57:01.172173 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-68fgb I0424 02:57:01.172175 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-5xngg after 0 failed attempt(s) I0424 02:57:01.172178 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-rpx7j I0424 02:57:01.172175 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-xsmjm I0424 02:57:01.172192 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-rpx7j I0424 02:57:01.172195 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-xsmjm I0424 02:57:01.172200 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-68fgb after 0 failed attempt(s) I0424 02:57:01.172213 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-xsmjm after 0 failed attempt(s) I0424 02:57:01.172151 3689 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-dk8km I0424 02:57:01.172232 3689 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-dk8km after 0 failed attempt(s) I0424 02:57:01.172177 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-zpz7f I0424 02:57:01.172220 3689 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-rpx7j] creating logical port openshift-multus_network-metrics-daemon-rpx7j for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:01.172254 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-zpz7f after 0 failed attempt(s) I0424 02:57:01.172162 3689 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-104.ec2.internal I0424 02:57:01.172267 3689 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-104.ec2.internal after 0 failed attempt(s) I0424 02:57:01.172149 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-x2m8g I0424 02:57:01.172279 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-x2m8g after 0 failed attempt(s) I0424 02:57:01.172164 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-104.ec2.internal I0424 02:57:01.172288 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-104.ec2.internal I0424 02:57:01.172163 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-m24hr I0424 02:57:01.172297 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-104.ec2.internal after 0 failed attempt(s) I0424 02:57:01.172301 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-m24hr I0424 02:57:01.172330 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-m24hr after 0 failed attempt(s) I0424 02:57:01.172173 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-n29gm I0424 02:57:01.172358 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-n29gm I0424 02:57:01.172370 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-n29gm after 0 failed attempt(s) I0424 02:57:01.172180 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-bz9v7 I0424 02:57:01.172381 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-bz9v7 I0424 02:57:01.172410 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-bz9v7 after 0 failed attempt(s) I0424 02:57:01.172144 3689 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-9q6zq I0424 02:57:01.172421 3689 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-9q6zq I0424 02:57:01.172466 3689 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-9q6zq] creating logical port openshift-network-diagnostics_network-check-target-9q6zq for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:01.173099 3689 pods.go:271] [openshift-network-diagnostics/network-check-target-9q6zq] addLogicalPort took 662.829µs, libovsdb time 226.129µs I0424 02:57:01.173111 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-9q6zq after 0 failed attempt(s) I0424 02:57:01.173190 3689 pods.go:271] [openshift-multus/network-metrics-daemon-rpx7j] addLogicalPort took 978.537µs, libovsdb time 283.434µs I0424 02:57:01.173196 3689 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-rpx7j after 0 failed attempt(s) I0424 02:57:01.187015 3689 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0424 02:57:01.198121 3689 iptables.go:108] Creating table: mangle chain: OUTPUT I0424 02:57:01.201262 3689 iptables.go:108] Creating table: nat chain: OUTPUT I0424 02:57:01.234163 3689 iptables.go:108] Creating table: nat chain: PREROUTING I0424 02:57:01.283899 3689 iptables.go:108] Creating table: filter chain: FORWARD I0424 02:57:01.299396 3689 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0424 02:57:01.299525 3689 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0424 02:57:01.299571 3689 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0424 02:57:01.299645 3689 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0424 02:57:01.299701 3689 gateway_init.go:325] Gateway and management port readiness took 1.184618347s I0424 02:57:01.299729 3689 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0424 02:57:01.299769 3689 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0424 02:57:01.299785 3689 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0424 02:57:01.299794 3689 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0424 02:57:01.299850 3689 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0424 02:57:01.299923 3689 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 47.34µs I0424 02:57:01.300217 3689 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0424 02:57:01.300242 3689 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.703µs I0424 02:57:01.300249 3689 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 33.385µs I0424 02:57:01.300255 3689 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0424 02:57:01.300265 3689 nad_controller.go:379] [node-nad-controller NAD controller]: started I0424 02:57:01.300274 3689 default_node_network_controller.go:899] Starting the default node network controller I0424 02:57:01.300280 3689 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0424 02:57:01.300310 3689 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 14.489µs I0424 02:57:01.300480 3689 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0424 02:57:01.300488 3689 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0424 02:57:01.300530 3689 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c32320 0xc0002ed7a0 0xc0002ed810 0xc0002ed8f0 0xc0002ed960 0xc0002ed9d0 0xc0002eda40 0xc000c780f0 0xc0002edb20 0xc0002edb90 0xc000c78320 0xc006593340 0xc0002edc00 map[0x2ff9080:0xc0023a1ce0 0x2ff95c0:0xc0023a1e60 0x2ffa2e0:0xc002592fc0 0x2ffa580:0xc0023a1c80 0x2ffad60:0xc0023a1d40 0x2ffb000:0xc0023a1da0 0x2ffb2a0:0xc0023a1e00 0x2ffba80:0xc0023a1ec0 0x2ffbd20:0xc0023a1f20 0x3026980:0xc00108dec0 0x3026d20:0xc00108df20 0x3027800:0xc000c74300 0x3029160:0xc000c74600 0x30298a0:0xc003c90180 0x302a720:0xc00108de60] 0xc000158cb0 0} I0424 02:57:01.300602 3689 gateway.go:151] Starting gateway service sync I0424 02:57:01.301144 3689 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0424 02:57:01.303313 3689 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0424 02:57:01.338447 3689 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0424 02:57:01.339589 3689 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0424 02:57:01.340709 3689 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0424 02:57:01.372501 3689 gateway.go:168] Gateway service sync done. Time taken: 71.889764ms I0424 02:57:01.373110 3689 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31256 I0424 02:57:01.373393 3689 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c32320 0xc0002ed7a0 0xc0002ed810 0xc0002ed8f0 0xc0002ed960 0xc0002ed9d0 0xc0002eda40 0xc000c780f0 0xc0002edb20 0xc0002edb90 0xc000c78320 0xc006593340 0xc0002edc00 map[0x2ff9080:0xc0023a1ce0 0x2ff95c0:0xc0023a1e60 0x2ffa2e0:0xc002592fc0 0x2ffa580:0xc0023a1c80 0x2ffad60:0xc0023a1d40 0x2ffb000:0xc0023a1da0 0x2ffb2a0:0xc0023a1e00 0x2ffba80:0xc0023a1ec0 0x2ffbd20:0xc0023a1f20 0x3026980:0xc00108dec0 0x3026d20:0xc00108df20 0x3027800:0xc000c74300 0x3029160:0xc000c74600 0x30298a0:0xc003c90180 0x302a720:0xc00108de60] 0xc000158cb0 0} I0424 02:57:01.373891 3689 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c32320 0xc0002ed7a0 0xc0002ed810 0xc0002ed8f0 0xc0002ed960 0xc0002ed9d0 0xc0002eda40 0xc000c780f0 0xc0002edb20 0xc0002edb90 0xc000c78320 0xc006593340 0xc0002edc00 map[0x2ff9080:0xc0023a1ce0 0x2ff95c0:0xc0023a1e60 0x2ffa2e0:0xc002592fc0 0x2ffa580:0xc0023a1c80 0x2ffad60:0xc0023a1d40 0x2ffb000:0xc0023a1da0 0x2ffb2a0:0xc0023a1e00 0x2ffba80:0xc0023a1ec0 0x2ffbd20:0xc0023a1f20 0x3026980:0xc00108dec0 0x3026d20:0xc00108df20 0x3027800:0xc000c74300 0x3029160:0xc000c74600 0x30298a0:0xc003c90180 0x302a720:0xc00108de60] 0xc000158cb0 0} I0424 02:57:01.373926 3689 gateway.go:532] Reconciling gateway with updates I0424 02:57:01.378390 3689 gateway.go:339] Spawning Conntrack Rule Check Thread I0424 02:57:01.378444 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0424 02:57:01.378464 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0424 02:57:01.378519 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0424 02:57:01.378532 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0424 02:57:01.378542 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0424 02:57:01.378542 3689 default_node_network_controller.go:944] Gateway and management port readiness took 78.042887ms I0424 02:57:01.378550 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0424 02:57:01.378559 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0424 02:57:01.378555 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0424 02:57:01.378572 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0424 02:57:01.378595 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0424 02:57:01.378613 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0424 02:57:01.378618 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0424 02:57:01.378627 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0424 02:57:01.378631 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0424 02:57:01.378624 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0424 02:57:01.378544 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0424 02:57:01.378638 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0424 02:57:01.378619 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0424 02:57:01.378645 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0424 02:57:01.378654 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0424 02:57:01.378662 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0424 02:57:01.378597 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0424 02:57:01.378640 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0424 02:57:01.378674 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0424 02:57:01.378675 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0424 02:57:01.378678 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0424 02:57:01.378681 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0424 02:57:01.378691 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0424 02:57:01.378644 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0424 02:57:01.378695 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0424 02:57:01.378705 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0424 02:57:01.378667 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0424 02:57:01.378719 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0424 02:57:01.378736 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0424 02:57:01.378441 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0424 02:57:01.378750 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0424 02:57:01.378757 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0424 02:57:01.378760 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0424 02:57:01.378762 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0424 02:57:01.378763 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0424 02:57:01.378618 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0424 02:57:01.378773 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0424 02:57:01.378775 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0424 02:57:01.378777 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0424 02:57:01.378780 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0424 02:57:01.378783 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0424 02:57:01.378784 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0424 02:57:01.378787 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0424 02:57:01.378791 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0424 02:57:01.378796 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0424 02:57:01.378800 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0424 02:57:01.378802 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0424 02:57:01.378804 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0424 02:57:01.378804 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0424 02:57:01.378806 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0424 02:57:01.378818 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0424 02:57:01.378820 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0424 02:57:01.378822 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0424 02:57:01.378826 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0424 02:57:01.378827 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0424 02:57:01.378836 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0424 02:57:01.378836 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0424 02:57:01.378842 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0424 02:57:01.378848 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0424 02:57:01.378850 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0424 02:57:01.378856 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0424 02:57:01.378855 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0424 02:57:01.378804 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0424 02:57:01.378813 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0424 02:57:01.378757 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0424 02:57:01.378867 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0424 02:57:01.378879 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0424 02:57:01.378882 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0424 02:57:01.378890 3689 port_claim.go:71] Svc openshift-ingress/router-default: port 32179 is already open, no action needed I0424 02:57:01.378896 3689 port_claim.go:71] Svc openshift-ingress/router-default: port 30544 is already open, no action needed I0424 02:57:01.378890 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0424 02:57:01.378921 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0424 02:57:01.378754 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0424 02:57:01.378819 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0424 02:57:01.378947 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0424 02:57:01.378987 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0424 02:57:01.378656 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0424 02:57:01.379003 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0424 02:57:01.379015 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0424 02:57:01.378850 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0424 02:57:01.378792 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0424 02:57:01.378775 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0424 02:57:01.378740 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0424 02:57:01.379069 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0424 02:57:01.379079 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0424 02:57:01.378682 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0424 02:57:01.378693 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0424 02:57:01.378759 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0424 02:57:01.379139 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0424 02:57:01.379155 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0424 02:57:01.378774 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0424 02:57:01.378603 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0424 02:57:01.378778 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0424 02:57:01.379219 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0424 02:57:01.378775 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0424 02:57:01.379267 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0424 02:57:01.378788 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0424 02:57:01.379319 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0424 02:57:01.378830 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0424 02:57:01.379346 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0424 02:57:01.379352 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0424 02:57:01.378851 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0424 02:57:01.378853 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0424 02:57:01.379395 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0424 02:57:01.378860 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0424 02:57:01.378862 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0424 02:57:01.379474 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0424 02:57:01.378861 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0424 02:57:01.378869 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0424 02:57:01.378931 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0424 02:57:01.378933 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0424 02:57:01.378930 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0424 02:57:01.379524 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0424 02:57:01.379742 3689 node_ip_handler_linux.go:160] Node IP manager is running W0424 02:57:01.414412 3689 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0424 02:57:01.414611 3689 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0424 02:57:01.414754 3689 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0424 02:57:01.423082 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0424 02:57:01.423090 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0424 02:57:01.423115 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0424 02:57:01.423123 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0424 02:57:01.423130 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0424 02:57:01.423138 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0424 02:57:01.423143 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0424 02:57:01.423148 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0424 02:57:01.423153 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0424 02:57:01.423159 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0424 02:57:01.423163 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0424 02:57:01.423169 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0424 02:57:01.459856 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0424 02:57:01.478037 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0424 02:57:01.478061 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0424 02:57:01.478077 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0424 02:57:03.384071 3689 default_node_network_controller.go:1380] Starting node controller node sync W0424 02:57:03.384101 3689 egressip_healthcheck.go:76] Health checking using insecure connection I0424 02:57:03.384148 3689 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0424 02:57:03.395489 3689 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.39134ms I0424 02:57:03.395614 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-137.ec2.internal", IP: 10.0.2.137 to PMTUD blocking rules I0424 02:57:03.395630 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-137.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0424 02:57:03.401435 3689 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0424 02:57:03.401738 3689 egressservice_node.go:93] Setting up event handlers for Egress Services I0424 02:57:03.401847 3689 egressservice_node.go:183] Starting Egress Services Controller I0424 02:57:03.401867 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0424 02:57:03.401878 3689 shared_informer.go:356] "Caches are synced" controller="egressservices" I0424 02:57:03.401886 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0424 02:57:03.401894 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0424 02:57:03.401900 3689 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0424 02:57:03.401904 3689 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0424 02:57:03.401908 3689 egressservice_node.go:197] Repairing Egress Services I0424 02:57:03.411490 3689 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0424 02:57:03.411500 3689 external_controller.go:278] Starting Admin Policy Based Route Controller I0424 02:57:03.411702 3689 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0424 02:57:03.411714 3689 link_network_manager.go:119] Link manager is running I0424 02:57:03.411723 3689 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0424 02:57:03.411893 3689 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0424 02:57:03.412026 3689 vrf_manager.go:128] VRF manager is running I0424 02:57:03.412095 3689 node_controller_manager.go:437] Removing flows to drop GARP I0424 02:57:03.412103 3689 gateway.go:532] Reconciling gateway with updates I0424 02:57:03.435435 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0424 02:57:03.435451 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0424 02:57:03.435456 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0424 02:57:03.435462 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0424 02:57:03.435463 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0424 02:57:03.435470 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0424 02:57:03.435475 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0424 02:57:03.435474 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0424 02:57:03.435486 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0424 02:57:03.435491 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0424 02:57:03.435501 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0424 02:57:03.435521 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0424 02:57:03.435443 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0424 02:57:03.435525 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0424 02:57:03.435529 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0424 02:57:03.435541 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0424 02:57:03.435551 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0424 02:57:03.435557 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0424 02:57:03.435555 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0424 02:57:03.435463 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0424 02:57:03.435577 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0424 02:57:03.435579 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0424 02:57:03.435583 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0424 02:57:03.435587 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0424 02:57:03.435590 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0424 02:57:03.435549 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0424 02:57:03.435596 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0424 02:57:03.435602 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0424 02:57:03.435606 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0424 02:57:03.435610 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0424 02:57:03.435613 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0424 02:57:03.435614 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0424 02:57:03.435620 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0424 02:57:03.435622 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0424 02:57:03.435536 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0424 02:57:03.435634 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0424 02:57:03.435487 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0424 02:57:03.435638 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0424 02:57:03.435465 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0424 02:57:03.435652 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0424 02:57:03.435651 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0424 02:57:03.435659 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0424 02:57:03.435664 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0424 02:57:03.435665 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0424 02:57:03.435668 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0424 02:57:03.435673 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0424 02:57:03.435674 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0424 02:57:03.435677 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0424 02:57:03.435679 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0424 02:57:03.435684 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0424 02:57:03.435690 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0424 02:57:03.435695 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0424 02:57:03.435700 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0424 02:57:03.435701 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0424 02:57:03.435701 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0424 02:57:03.435711 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0424 02:57:03.435705 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0424 02:57:03.435711 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0424 02:57:03.435716 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0424 02:57:03.435718 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0424 02:57:03.435716 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0424 02:57:03.435720 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0424 02:57:03.435726 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0424 02:57:03.435730 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0424 02:57:03.435732 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0424 02:57:03.435730 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0424 02:57:03.435734 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0424 02:57:03.435738 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0424 02:57:03.435740 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0424 02:57:03.435741 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0424 02:57:03.435717 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0424 02:57:03.435750 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0424 02:57:03.435730 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0424 02:57:03.435757 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0424 02:57:03.435767 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0424 02:57:03.435770 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0424 02:57:03.435561 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0424 02:57:03.435781 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0424 02:57:03.435701 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0424 02:57:03.435620 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0424 02:57:03.435607 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0424 02:57:03.435691 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0424 02:57:03.435870 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0424 02:57:03.435680 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0424 02:57:03.435874 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0424 02:57:03.435475 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0424 02:57:03.435912 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0424 02:57:03.435919 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0424 02:57:03.435584 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0424 02:57:03.435497 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0424 02:57:03.435593 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0424 02:57:03.435573 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0424 02:57:03.435988 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0424 02:57:03.435628 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0424 02:57:03.436020 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0424 02:57:03.436025 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0424 02:57:03.435628 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0424 02:57:03.436038 3689 port_claim.go:71] Svc openshift-ingress/router-default: port 32179 is already open, no action needed I0424 02:57:03.436046 3689 port_claim.go:71] Svc openshift-ingress/router-default: port 30544 is already open, no action needed I0424 02:57:03.435638 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0424 02:57:03.436091 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0424 02:57:03.435640 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0424 02:57:03.436103 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0424 02:57:03.435637 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0424 02:57:03.436134 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0424 02:57:03.435643 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0424 02:57:03.435644 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0424 02:57:03.435652 3689 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0424 02:57:03.436210 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0424 02:57:03.436217 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0424 02:57:03.435653 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0424 02:57:03.435691 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0424 02:57:03.435693 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0424 02:57:03.435703 3689 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0424 02:57:03.435749 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0424 02:57:03.477165 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0424 02:57:03.477186 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0424 02:57:03.477192 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0424 02:57:03.477198 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0424 02:57:03.477204 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0424 02:57:03.477209 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0424 02:57:03.477216 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0424 02:57:03.515762 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0424 02:57:03.515789 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0424 02:57:03.515806 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0424 02:57:03.515814 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0424 02:57:03.515820 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0424 02:57:03.515826 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0424 02:57:03.558492 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0424 02:57:03.558499 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0424 02:57:03.558532 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0424 02:57:03.558540 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0424 02:57:03.558545 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0424 02:57:03.558552 3689 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) W0424 02:57:04.219873 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0424 02:57:04.804646 3689 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-6b6df6cddf-dd7m7 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-6b6df6cddf-dd7m7: no pod IPs found W0424 02:57:04.822031 3689 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-7bb69fc5c4-rrnf8 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-7bb69fc5c4-rrnf8: no pod IPs found I0424 02:57:04.972942 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-104.ec2.internal I0424 02:57:04.972974 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-104.ec2.internal: 38.509µs I0424 02:57:05.007410 3689 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-8648f5cc7c-n6qpc] creating logical port openshift-ingress_router-default-8648f5cc7c-n6qpc for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:05.007569 3689 kube.go:256] Updating pod openshift-ingress/router-default-8648f5cc7c-n6qpc I0424 02:57:05.030449 3689 pod.go:62] [openshift-ingress/router-default-8648f5cc7c-n6qpc] pod update took 22.91993ms I0424 02:57:05.030472 3689 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-8648f5cc7c-n6qpc] addLogicalPort annotation time took 22.978441ms I0424 02:57:05.031803 3689 pods.go:271] [openshift-ingress/router-default-8648f5cc7c-n6qpc] addLogicalPort took 24.403454ms, libovsdb time 1.032509ms I0424 02:57:05.031860 3689 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-64d9447c48-rwxvb] creating logical port openshift-image-registry_image-registry-64d9447c48-rwxvb for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:05.031941 3689 kube.go:256] Updating pod openshift-image-registry/image-registry-64d9447c48-rwxvb I0424 02:57:05.037400 3689 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-9k5xq] creating logical port openshift-ingress-canary_ingress-canary-9k5xq for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:05.037554 3689 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-9k5xq I0424 02:57:05.050693 3689 pod.go:62] [openshift-image-registry/image-registry-64d9447c48-rwxvb] pod update took 18.783064ms I0424 02:57:05.050712 3689 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-64d9447c48-rwxvb] addLogicalPort annotation time took 18.805705ms I0424 02:57:05.051684 3689 pods.go:271] [openshift-image-registry/image-registry-64d9447c48-rwxvb] addLogicalPort took 19.832477ms, libovsdb time 754.514µs I0424 02:57:05.054862 3689 pod.go:62] [openshift-ingress-canary/ingress-canary-9k5xq] pod update took 17.355799ms I0424 02:57:05.054885 3689 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-9k5xq] addLogicalPort annotation time took 17.382786ms I0424 02:57:05.055889 3689 pods.go:271] [openshift-ingress-canary/ingress-canary-9k5xq] addLogicalPort took 18.50031ms, libovsdb time 777.362µs I0424 02:57:05.058292 3689 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-frjrt] creating logical port openshift-dns_dns-default-frjrt for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:05.058373 3689 kube.go:256] Updating pod openshift-dns/dns-default-frjrt I0424 02:57:05.074253 3689 pod.go:62] [openshift-dns/dns-default-frjrt] pod update took 15.895935ms I0424 02:57:05.074274 3689 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-frjrt] addLogicalPort annotation time took 15.91675ms I0424 02:57:05.075369 3689 pods.go:271] [openshift-dns/dns-default-frjrt] addLogicalPort took 17.088759ms, libovsdb time 759.884µs I0424 02:57:05.451400 3689 cni.go:452] [openshift-image-registry/image-registry-64d9447c48-rwxvb 7d9fbc378c889b2f616d8e308fc0a97a70e856cd1bb0358d299ef0b9d0f59fac network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-64d9447c48-rwxvb 7d9fbc378c889b2f616d8e308fc0a97a70e856cd1bb0358d299ef0b9d0f59fac network default NAD default NAD key ] I0424 02:57:05.460672 3689 cni.go:452] [openshift-ingress-canary/ingress-canary-9k5xq 7f0ad32c915d79c5e0e0a01613ef5295c9d1da9fb2d77fa18a4ca283f1443a78 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-9k5xq 7f0ad32c915d79c5e0e0a01613ef5295c9d1da9fb2d77fa18a4ca283f1443a78 network default NAD default NAD key ] I0424 02:57:05.487609 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-64d9447c48-rwxvb, hostIfaceName: 7d9fbc378c889b2, network: default, NAD default, SandboxID: "7d9fbc378c889b2f616d8e308fc0a97a70e856cd1bb0358d299ef0b9d0f59fac", PCI device ID: , UID: "94046b9d-0d87-42b6-bc1c-1ab85ef2ed23", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0424 02:57:05.487756 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-9k5xq, hostIfaceName: 7f0ad32c915d79c, network: default, NAD default, SandboxID: "7f0ad32c915d79c5e0e0a01613ef5295c9d1da9fb2d77fa18a4ca283f1443a78", PCI device ID: , UID: "c0073dd2-ddf9-4149-8cf4-6eedaad47dbf", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0424 02:57:05.498386 3689 cni.go:452] [openshift-dns/dns-default-frjrt 4bbb1e63e0bba4722b3783478bc2f8b306a8f1a220b184b164b9abc71c8e3a23 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-frjrt 4bbb1e63e0bba4722b3783478bc2f8b306a8f1a220b184b164b9abc71c8e3a23 network default NAD default NAD key ] I0424 02:57:05.510866 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-frjrt, hostIfaceName: 4bbb1e63e0bba47, network: default, NAD default, SandboxID: "4bbb1e63e0bba4722b3783478bc2f8b306a8f1a220b184b164b9abc71c8e3a23", PCI device ID: , UID: "fe9dfe1e-d3e5-4363-b8c6-b5057634a073", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0424 02:57:05.539953 3689 cni.go:473] [openshift-ingress-canary/ingress-canary-9k5xq 7f0ad32c915d79c5e0e0a01613ef5295c9d1da9fb2d77fa18a4ca283f1443a78 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-9k5xq 7f0ad32c915d79c5e0e0a01613ef5295c9d1da9fb2d77fa18a4ca283f1443a78 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:e5:04:8f:17:76\",\"name\":\"7f0ad32c915d79c\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4531bb85-43a7-4d5d-87d9-f238dde2e455\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:05.546053 3689 cni.go:473] [openshift-image-registry/image-registry-64d9447c48-rwxvb 7d9fbc378c889b2f616d8e308fc0a97a70e856cd1bb0358d299ef0b9d0f59fac network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-64d9447c48-rwxvb 7d9fbc378c889b2f616d8e308fc0a97a70e856cd1bb0358d299ef0b9d0f59fac network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:df:4b:08:0b:72\",\"name\":\"7d9fbc378c889b2\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2f4e83fd-0fcd-41e3-934f-9687f137ec66\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:05.558938 3689 cni.go:473] [openshift-dns/dns-default-frjrt 4bbb1e63e0bba4722b3783478bc2f8b306a8f1a220b184b164b9abc71c8e3a23 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-frjrt 4bbb1e63e0bba4722b3783478bc2f8b306a8f1a220b184b164b9abc71c8e3a23 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:74:36:a2:bb:b6\",\"name\":\"4bbb1e63e0bba47\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90f19ad2-1573-4e4b-b0a1-51e144985792\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:06.488301 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-139.ec2.internal I0424 02:57:06.488377 3689 default_network_controller.go:1025] Node "ip-10-0-1-139.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0424 02:57:06.488401 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-139.ec2.internal: 106.772µs I0424 02:57:06.488341 3689 egressfirewall.go:723] Syncing node "ip-10-0-1-139.ec2.internal" for egress firewall I0424 02:57:09.397243 3689 cni.go:452] [openshift-multus/network-metrics-daemon-rpx7j 354383eac4e8985957071a464df26160481dda910fa22b30483cecf17bb9cc80 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-rpx7j 354383eac4e8985957071a464df26160481dda910fa22b30483cecf17bb9cc80 network default NAD default NAD key ] I0424 02:57:09.409229 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-rpx7j, hostIfaceName: 354383eac4e8985, network: default, NAD default, SandboxID: "354383eac4e8985957071a464df26160481dda910fa22b30483cecf17bb9cc80", PCI device ID: , UID: "6ba0907c-0f95-45a6-996a-44d115ddedb6", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0424 02:57:09.457658 3689 cni.go:473] [openshift-multus/network-metrics-daemon-rpx7j 354383eac4e8985957071a464df26160481dda910fa22b30483cecf17bb9cc80 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-rpx7j 354383eac4e8985957071a464df26160481dda910fa22b30483cecf17bb9cc80 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:71:49:d0:67:bd\",\"name\":\"354383eac4e8985\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/75e4ca0f-5cbd-438c-8390-6a1a6ffeb9d7\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:09.608444 3689 cni.go:452] [openshift-network-diagnostics/network-check-target-9q6zq 171ffd7f80988c74746a59e22ff1bb707351b059eba6206eb9467f78c83412f7 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-9q6zq 171ffd7f80988c74746a59e22ff1bb707351b059eba6206eb9467f78c83412f7 network default NAD default NAD key ] I0424 02:57:09.620527 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-9q6zq, hostIfaceName: 171ffd7f80988c7, network: default, NAD default, SandboxID: "171ffd7f80988c74746a59e22ff1bb707351b059eba6206eb9467f78c83412f7", PCI device ID: , UID: "e6a2db47-59ff-44be-aaed-6dfc7f865a2f", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0424 02:57:09.671565 3689 cni.go:473] [openshift-network-diagnostics/network-check-target-9q6zq 171ffd7f80988c74746a59e22ff1bb707351b059eba6206eb9467f78c83412f7 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-9q6zq 171ffd7f80988c74746a59e22ff1bb707351b059eba6206eb9467f78c83412f7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:2d:ab:80:45:f6\",\"name\":\"171ffd7f80988c7\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0d8af278-58dd-4e87-a35e-7c31eca534c0\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:13.363627 3689 cni.go:452] [openshift-ingress/router-default-8648f5cc7c-n6qpc 9d1bc0fd36abf099870e910f411e80d546395dbd4d26c6097e36691cdec15c44 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-8648f5cc7c-n6qpc 9d1bc0fd36abf099870e910f411e80d546395dbd4d26c6097e36691cdec15c44 network default NAD default NAD key ] I0424 02:57:13.373408 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-8648f5cc7c-n6qpc, hostIfaceName: 9d1bc0fd36abf09, network: default, NAD default, SandboxID: "9d1bc0fd36abf099870e910f411e80d546395dbd4d26c6097e36691cdec15c44", PCI device ID: , UID: "9da1e80c-edad-4abe-8a37-9dff810ae197", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0424 02:57:13.417710 3689 cni.go:473] [openshift-ingress/router-default-8648f5cc7c-n6qpc 9d1bc0fd36abf099870e910f411e80d546395dbd4d26c6097e36691cdec15c44 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-8648f5cc7c-n6qpc 9d1bc0fd36abf099870e910f411e80d546395dbd4d26c6097e36691cdec15c44 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:83:91:77:29:ea\",\"name\":\"9d1bc0fd36abf09\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f1d208e9-a012-4112-9220-0112b4535c73\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:18.179344 3689 default_network_controller.go:1025] Node "ip-10-0-1-139.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0424 02:57:25.208239 3689 default_network_controller.go:1025] Node "ip-10-0-1-139.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0424 02:57:25.277317 3689 default_network_controller.go:1025] Node "ip-10-0-1-139.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0424 02:57:25.335522 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0424 02:57:25.420858 3689 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0424 02:57:25.425425 3689 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0424 02:57:25.438213 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lfpgq] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-lfpgq for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:25.438293 3689 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lfpgq I0424 02:57:25.463335 3689 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lfpgq] pod update took 25.064705ms I0424 02:57:25.463353 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lfpgq] addLogicalPort annotation time took 25.088973ms I0424 02:57:25.464631 3689 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lfpgq] addLogicalPort took 26.429755ms, libovsdb time 949.098µs W0424 02:57:25.553261 3689 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0424 02:57:25.571380 3689 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-d76ll] creating logical port openshift-console_downloads-5c54f4944f-d76ll for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:25.571459 3689 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-d76ll W0424 02:57:25.576403 3689 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-64d9447c48-rstmk IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-64d9447c48-rstmk: no pod IPs found I0424 02:57:25.598009 3689 pod.go:62] [openshift-console/downloads-5c54f4944f-d76ll] pod update took 26.572553ms I0424 02:57:25.598031 3689 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-d76ll] addLogicalPort annotation time took 26.604963ms I0424 02:57:25.599031 3689 pods.go:271] [openshift-console/downloads-5c54f4944f-d76ll] addLogicalPort took 27.664028ms, libovsdb time 770.607µs I0424 02:57:25.695245 3689 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-wk2pp] creating logical port openshift-insights_insights-runtime-extractor-wk2pp for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:25.695317 3689 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-wk2pp I0424 02:57:25.721415 3689 pod.go:62] [openshift-insights/insights-runtime-extractor-wk2pp] pod update took 26.12148ms I0424 02:57:25.721435 3689 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-wk2pp] addLogicalPort annotation time took 26.143011ms I0424 02:57:25.722648 3689 pods.go:271] [openshift-insights/insights-runtime-extractor-wk2pp] addLogicalPort took 27.411539ms, libovsdb time 975.783µs I0424 02:57:25.820988 3689 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lfpgq a3dc76c84ee721b2589bacae2270b29b2344093da568dd0aceca64a53d81ed09 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lfpgq a3dc76c84ee721b2589bacae2270b29b2344093da568dd0aceca64a53d81ed09 network default NAD default NAD key ] I0424 02:57:25.830639 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-lfpgq, hostIfaceName: a3dc76c84ee721b, network: default, NAD default, SandboxID: "a3dc76c84ee721b2589bacae2270b29b2344093da568dd0aceca64a53d81ed09", PCI device ID: , UID: "3cd41f1a-692c-4a09-867f-b42fd40c9d83", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0424 02:57:25.874525 3689 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lfpgq a3dc76c84ee721b2589bacae2270b29b2344093da568dd0aceca64a53d81ed09 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-lfpgq a3dc76c84ee721b2589bacae2270b29b2344093da568dd0aceca64a53d81ed09 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:3c:41:4b:64:8d\",\"name\":\"a3dc76c84ee721b\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ad11198a-fe70-4e92-9d88-459c05a2cbc5\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:25.984869 3689 cni.go:452] [openshift-console/downloads-5c54f4944f-d76ll a897978fb8fac04ace3e1693f475c7075d523e1a6a8578ec234e99b7c30d1ba4 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-d76ll a897978fb8fac04ace3e1693f475c7075d523e1a6a8578ec234e99b7c30d1ba4 network default NAD default NAD key ] I0424 02:57:26.014693 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-d76ll, hostIfaceName: a897978fb8fac04, network: default, NAD default, SandboxID: "a897978fb8fac04ace3e1693f475c7075d523e1a6a8578ec234e99b7c30d1ba4", PCI device ID: , UID: "eba67885-5c36-4838-a6a0-bc2ff336de65", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0424 02:57:26.057222 3689 cni.go:473] [openshift-console/downloads-5c54f4944f-d76ll a897978fb8fac04ace3e1693f475c7075d523e1a6a8578ec234e99b7c30d1ba4 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-d76ll a897978fb8fac04ace3e1693f475c7075d523e1a6a8578ec234e99b7c30d1ba4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:71:b8:8c:1e:3a\",\"name\":\"a897978fb8fac04\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/20a565b4-f07e-4ee1-8010-5286b050da63\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:26.081278 3689 cni.go:452] [openshift-insights/insights-runtime-extractor-wk2pp a8d557e1a1e5709941ba57cb15725066a10e2dcc8929005a0ae0be18b54ecea7 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-wk2pp a8d557e1a1e5709941ba57cb15725066a10e2dcc8929005a0ae0be18b54ecea7 network default NAD default NAD key ] I0424 02:57:26.090232 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-wk2pp, hostIfaceName: a8d557e1a1e5709, network: default, NAD default, SandboxID: "a8d557e1a1e5709941ba57cb15725066a10e2dcc8929005a0ae0be18b54ecea7", PCI device ID: , UID: "19fc382d-78b6-473a-bdcc-c8f690fbc18a", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0424 02:57:26.133417 3689 cni.go:473] [openshift-insights/insights-runtime-extractor-wk2pp a8d557e1a1e5709941ba57cb15725066a10e2dcc8929005a0ae0be18b54ecea7 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-wk2pp a8d557e1a1e5709941ba57cb15725066a10e2dcc8929005a0ae0be18b54ecea7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:76:42:3f:39:bb\",\"name\":\"a8d557e1a1e5709\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fefc1e23-6ee5-4e44-ac8e-570c131096c4\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:26.551749 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-139.ec2.internal in network "default" I0424 02:57:26.551822 3689 node_tracker.go:235] Node ip-10-0-1-139.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-139.ec2.internal" I0424 02:57:26.551824 3689 default_network_controller.go:1025] Node "ip-10-0-1-139.ec2.internal" in remote zone "ip-10-0-1-139.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0424 02:57:26.551840 3689 node_tracker.go:169] Node ip-10-0-1-139.ec2.internal switch + router changed, syncing services in network "default" I0424 02:57:26.551855 3689 services_controller.go:574] Full service sync requested for network=default W0424 02:57:26.552065 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0424 02:57:26.552483 3689 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0424 02:57:26.587523 3689 default_network_controller.go:1025] Node "ip-10-0-1-139.ec2.internal" in remote zone "ip-10-0-1-139.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0424 02:57:26.728607 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-139.ec2.internal in network "default" I0424 02:57:26.728640 3689 default_network_controller.go:1025] Node "ip-10-0-1-139.ec2.internal" in remote zone "ip-10-0-1-139.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0424 02:57:26.728662 3689 node_tracker.go:169] Node ip-10-0-1-139.ec2.internal switch + router changed, syncing services in network "default" I0424 02:57:26.728674 3689 services_controller.go:574] Full service sync requested for network=default W0424 02:57:26.729067 3689 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0424 02:57:26.729180 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0424 02:57:26.729680 3689 zone_ic_handler.go:195] Time taken to create transit switch: 304.602µs I0424 02:57:26.729699 3689 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-139.ec2.internal for the network default I0424 02:57:26.731181 3689 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-139.ec2.internal" on network "default" took: 1.824872ms I0424 02:57:26.863155 3689 base_network_controller_pods.go:487] [default/openshift-console/console-84465d948b-zmr6m] creating logical port openshift-console_console-84465d948b-zmr6m for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:26.863297 3689 kube.go:256] Updating pod openshift-console/console-84465d948b-zmr6m I0424 02:57:26.878722 3689 pod.go:62] [openshift-console/console-84465d948b-zmr6m] pod update took 15.452438ms I0424 02:57:26.878745 3689 base_network_controller_pods.go:951] [default/openshift-console/console-84465d948b-zmr6m] addLogicalPort annotation time took 15.477407ms I0424 02:57:26.880081 3689 pods.go:271] [openshift-console/console-84465d948b-zmr6m] addLogicalPort took 16.967622ms, libovsdb time 921.038µs I0424 02:57:27.280473 3689 egressfirewall.go:723] Syncing node "ip-10-0-1-139.ec2.internal" for egress firewall I0424 02:57:27.280527 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-139.ec2.internal I0424 02:57:27.280547 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-139.ec2.internal: 40.526µs I0424 02:57:27.280555 3689 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-139.ec2.internal in network "default" I0424 02:57:27.280592 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-139.ec2.internal", IP: 10.0.1.139 to PMTUD blocking rules I0424 02:57:27.280606 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-139.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0424 02:57:27.280606 3689 node_tracker.go:169] Node ip-10-0-1-139.ec2.internal switch + router changed, syncing services in network "default" I0424 02:57:27.280621 3689 services_controller.go:574] Full service sync requested for network=default W0424 02:57:27.280954 3689 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0424 02:57:27.281035 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0424 02:57:27.286550 3689 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-139.ec2.internal. Updating no re-route policies I0424 02:57:27.301444 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-139.ec2.internal", IP: 10.0.1.139 to PMTUD blocking rules I0424 02:57:27.301459 3689 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-139.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0424 02:57:27.682950 3689 cni.go:452] [openshift-console/console-84465d948b-zmr6m d2708496a134a9c9857a5df897fe6059f9dbf4670322a759dd563a995601aaa5 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-84465d948b-zmr6m d2708496a134a9c9857a5df897fe6059f9dbf4670322a759dd563a995601aaa5 network default NAD default NAD key ] I0424 02:57:27.692096 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-84465d948b-zmr6m, hostIfaceName: d2708496a134a9c, network: default, NAD default, SandboxID: "d2708496a134a9c9857a5df897fe6059f9dbf4670322a759dd563a995601aaa5", PCI device ID: , UID: "d145015a-5c7b-4e89-bebb-1954e675933f", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] W0424 02:57:27.695518 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0424 02:57:27.743684 3689 cni.go:473] [openshift-console/console-84465d948b-zmr6m d2708496a134a9c9857a5df897fe6059f9dbf4670322a759dd563a995601aaa5 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-84465d948b-zmr6m d2708496a134a9c9857a5df897fe6059f9dbf4670322a759dd563a995601aaa5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:e9:50:28:fa:af\",\"name\":\"d2708496a134a9c\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fcdd00f6-2a10-4892-b29e-6f87050efb51\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 02:57:27.937263 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0424 02:57:28.809643 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0424 02:57:29.583530 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-jvqph] creating logical port openshift-monitoring_prometheus-operator-76b589d668-jvqph for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:29.583655 3689 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-jvqph I0424 02:57:29.604475 3689 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-jvqph] pod update took 20.860075ms I0424 02:57:29.604499 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-jvqph] addLogicalPort annotation time took 20.890296ms I0424 02:57:29.605701 3689 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-jvqph] addLogicalPort took 22.208548ms, libovsdb time 837.034µs I0424 02:57:31.356085 3689 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-jvqph 0292e9e8b908388a24ed98945860739de6d60a39a197514307d4b319fa63968d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-jvqph 0292e9e8b908388a24ed98945860739de6d60a39a197514307d4b319fa63968d network default NAD default NAD key ] I0424 02:57:31.364935 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-jvqph, hostIfaceName: 0292e9e8b908388, network: default, NAD default, SandboxID: "0292e9e8b908388a24ed98945860739de6d60a39a197514307d4b319fa63968d", PCI device ID: , UID: "f431d2ab-ae8f-49c8-b12e-721afe60226d", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0424 02:57:31.384033 3689 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-139.ec2.internal I0424 02:57:31.384070 3689 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-139.ec2.internal: 47.81µs I0424 02:57:31.413894 3689 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-jvqph 0292e9e8b908388a24ed98945860739de6d60a39a197514307d4b319fa63968d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-jvqph 0292e9e8b908388a24ed98945860739de6d60a39a197514307d4b319fa63968d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:b9:85:7e:4e:6a\",\"name\":\"0292e9e8b908388\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5cd0506e-48df-4870-a769-24c0db61c134\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 02:57:31.792661 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0424 02:57:31.972393 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0424 02:57:34.088292 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0424 02:57:37.249776 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:37.249879 3689 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0424 02:57:37.270994 3689 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 21.155509ms I0424 02:57:37.271012 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 21.177486ms I0424 02:57:37.272456 3689 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 22.698773ms, libovsdb time 1.055301ms W0424 02:57:38.034124 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0424 02:57:38.034150 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0424 02:57:38.203052 3689 cni.go:452] [openshift-monitoring/alertmanager-main-0 e277b9c0a825e9abd203abca993ed1bd92d31da3249696de12e20343ef2c15e3 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 e277b9c0a825e9abd203abca993ed1bd92d31da3249696de12e20343ef2c15e3 network default NAD default NAD key ] I0424 02:57:38.212181 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: e277b9c0a825e9a, network: default, NAD default, SandboxID: "e277b9c0a825e9abd203abca993ed1bd92d31da3249696de12e20343ef2c15e3", PCI device ID: , UID: "72433e2d-2f56-4787-92c2-083ab19da308", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0424 02:57:38.260463 3689 cni.go:473] [openshift-monitoring/alertmanager-main-0 e277b9c0a825e9abd203abca993ed1bd92d31da3249696de12e20343ef2c15e3 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 e277b9c0a825e9abd203abca993ed1bd92d31da3249696de12e20343ef2c15e3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:5f:04:d1:f6:ff\",\"name\":\"e277b9c0a825e9a\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/36f12031-cdfe-4c32-b63b-faae4c82cfe3\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:39.312535 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-fdd4cddfc-9vh48] creating logical port openshift-monitoring_thanos-querier-fdd4cddfc-9vh48 for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:39.312628 3689 kube.go:256] Updating pod openshift-monitoring/thanos-querier-fdd4cddfc-9vh48 I0424 02:57:39.330728 3689 pod.go:62] [openshift-monitoring/thanos-querier-fdd4cddfc-9vh48] pod update took 18.130269ms I0424 02:57:39.330756 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-fdd4cddfc-9vh48] addLogicalPort annotation time took 18.155776ms I0424 02:57:39.332072 3689 pods.go:271] [openshift-monitoring/thanos-querier-fdd4cddfc-9vh48] addLogicalPort took 19.571187ms, libovsdb time 1.04784ms I0424 02:57:41.118816 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-pstfm] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-pstfm for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:41.118911 3689 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-pstfm I0424 02:57:41.137960 3689 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pstfm] pod update took 19.072804ms I0424 02:57:41.137983 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-pstfm] addLogicalPort annotation time took 19.099673ms I0424 02:57:41.139040 3689 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pstfm] addLogicalPort took 20.263373ms, libovsdb time 754.892µs I0424 02:57:41.205084 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-7dc5966c6b-l7lmc] creating logical port openshift-monitoring_metrics-server-7dc5966c6b-l7lmc for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:41.205253 3689 kube.go:256] Updating pod openshift-monitoring/metrics-server-7dc5966c6b-l7lmc I0424 02:57:41.219098 3689 pod.go:62] [openshift-monitoring/metrics-server-7dc5966c6b-l7lmc] pod update took 13.87984ms I0424 02:57:41.219125 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-7dc5966c6b-l7lmc] addLogicalPort annotation time took 13.910592ms I0424 02:57:41.220305 3689 pods.go:271] [openshift-monitoring/metrics-server-7dc5966c6b-l7lmc] addLogicalPort took 15.240555ms, libovsdb time 857.958µs W0424 02:57:41.719933 3689 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-86dc6b9fd4-zdrmp IP address from the namespace address-set, err: pod openshift-console/console-86dc6b9fd4-zdrmp: no pod IPs found W0424 02:57:41.781847 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 02:57:41.781870 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 02:57:41.781879 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0424 02:57:42.651670 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:42.651774 3689 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0424 02:57:42.685768 3689 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 34.027756ms I0424 02:57:42.685861 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 34.127928ms I0424 02:57:42.687319 3689 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 35.662116ms, libovsdb time 1.013165ms W0424 02:57:44.728462 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0424 02:57:45.751702 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0424 02:57:45.776611 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0424 02:57:45.817454 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 02:57:45.817470 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 02:57:46.023544 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0424 02:57:46.023961 3689 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pstfm f3f20addf090dbb35d732cc6a1bf91e5fe8d698cbd96cff97f3081dfd87e3ac1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pstfm f3f20addf090dbb35d732cc6a1bf91e5fe8d698cbd96cff97f3081dfd87e3ac1 network default NAD default NAD key ] I0424 02:57:46.024668 3689 cni.go:452] [openshift-monitoring/thanos-querier-fdd4cddfc-9vh48 ed326f800a7fbaa456e8c02dab94e1e88565b58fbc4a7289863ef471e9efad82 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-fdd4cddfc-9vh48 ed326f800a7fbaa456e8c02dab94e1e88565b58fbc4a7289863ef471e9efad82 network default NAD default NAD key ] I0424 02:57:46.025778 3689 cni.go:452] [openshift-monitoring/metrics-server-7dc5966c6b-l7lmc 8c1c10bc86d07bb9484545e174fc28eb4292529aefe1cae7a6210b9941d898df network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-7dc5966c6b-l7lmc 8c1c10bc86d07bb9484545e174fc28eb4292529aefe1cae7a6210b9941d898df network default NAD default NAD key ] I0424 02:57:46.029310 3689 cni.go:452] [openshift-monitoring/prometheus-k8s-1 6c877a84a908033dfdc06d0206dd847230e55b52b2b51f8e97509a50101d6e10 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 6c877a84a908033dfdc06d0206dd847230e55b52b2b51f8e97509a50101d6e10 network default NAD default NAD key ] I0424 02:57:46.044125 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-7dc5966c6b-l7lmc, hostIfaceName: 8c1c10bc86d07bb, network: default, NAD default, SandboxID: "8c1c10bc86d07bb9484545e174fc28eb4292529aefe1cae7a6210b9941d898df", PCI device ID: , UID: "18ee90d5-40f0-4ebc-a1ad-63f916652189", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0424 02:57:46.046204 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-pstfm, hostIfaceName: f3f20addf090dbb, network: default, NAD default, SandboxID: "f3f20addf090dbb35d732cc6a1bf91e5fe8d698cbd96cff97f3081dfd87e3ac1", PCI device ID: , UID: "de2470d8-145f-446e-a48c-b09d0dba9cff", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0424 02:57:46.058962 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-fdd4cddfc-9vh48, hostIfaceName: ed326f800a7fbaa, network: default, NAD default, SandboxID: "ed326f800a7fbaa456e8c02dab94e1e88565b58fbc4a7289863ef471e9efad82", PCI device ID: , UID: "ba9c8cb3-ca21-400c-a42b-67f1065cd45f", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0424 02:57:46.073050 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 6c877a84a908033, network: default, NAD default, SandboxID: "6c877a84a908033dfdc06d0206dd847230e55b52b2b51f8e97509a50101d6e10", PCI device ID: , UID: "d04af4d1-585c-4a43-ae75-d5b9526ca939", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0424 02:57:46.112295 3689 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pstfm f3f20addf090dbb35d732cc6a1bf91e5fe8d698cbd96cff97f3081dfd87e3ac1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-pstfm f3f20addf090dbb35d732cc6a1bf91e5fe8d698cbd96cff97f3081dfd87e3ac1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:f7:49:9f:5a:fb\",\"name\":\"f3f20addf090dbb\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be1265a9-57e3-44a9-8155-c86579bedb85\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:46.343738 3689 cni.go:473] [openshift-monitoring/metrics-server-7dc5966c6b-l7lmc 8c1c10bc86d07bb9484545e174fc28eb4292529aefe1cae7a6210b9941d898df network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-7dc5966c6b-l7lmc 8c1c10bc86d07bb9484545e174fc28eb4292529aefe1cae7a6210b9941d898df network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:01:79:12:01:19\",\"name\":\"8c1c10bc86d07bb\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8fcf41dc-6775-4644-8a7d-ee2842c1436a\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:46.343768 3689 cni.go:473] [openshift-monitoring/thanos-querier-fdd4cddfc-9vh48 ed326f800a7fbaa456e8c02dab94e1e88565b58fbc4a7289863ef471e9efad82 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-fdd4cddfc-9vh48 ed326f800a7fbaa456e8c02dab94e1e88565b58fbc4a7289863ef471e9efad82 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:2f:d5:09:af:ab\",\"name\":\"ed326f800a7fbaa\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/83990213-000a-4c35-bd46-34590cd96285\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:46.343771 3689 cni.go:473] [openshift-monitoring/prometheus-k8s-1 6c877a84a908033dfdc06d0206dd847230e55b52b2b51f8e97509a50101d6e10 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 6c877a84a908033dfdc06d0206dd847230e55b52b2b51f8e97509a50101d6e10 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:67:f8:28:66:fd\",\"name\":\"6c877a84a908033\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/17fd6c6a-3fdf-4623-865f-c7a0a738061a\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 02:57:46.878899 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 02:57:46.878995 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 02:57:46.879011 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 02:57:46.924903 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 02:57:46.924932 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 02:57:47.810968 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 02:57:47.811057 3689 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 02:57:47.811103 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 02:57:47.811111 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 02:57:48.894345 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0424 02:57:50.863217 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7bb69fc5c4-9pfrw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:57:50.863777 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7bb69fc5c4-9pfrw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:57:51.051340 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7bb69fc5c4-9pfrw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:57:51.051366 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7bb69fc5c4-9pfrw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:57:51.058859 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7bb69fc5c4-9pfrw I0424 02:57:51.058890 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7bb69fc5c4-9pfrw W0424 02:57:57.950752 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 02:57:57.950773 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 02:57:57.950781 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0424 02:57:57.950787 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0424 02:57:59.247738 3689 cni.go:452] [openshift-console/console-84465d948b-zmr6m d2708496a134a9c9857a5df897fe6059f9dbf4670322a759dd563a995601aaa5 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-84465d948b-zmr6m d2708496a134a9c9857a5df897fe6059f9dbf4670322a759dd563a995601aaa5 network default NAD default NAD key ] I0424 02:57:59.325627 3689 cni.go:473] [openshift-console/console-84465d948b-zmr6m d2708496a134a9c9857a5df897fe6059f9dbf4670322a759dd563a995601aaa5 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-84465d948b-zmr6m d2708496a134a9c9857a5df897fe6059f9dbf4670322a759dd563a995601aaa5 network default NAD default NAD key default], result "{}", err I0424 02:57:59.382687 3689 obj_retry.go:498] Detected object openshift-console/console-84465d948b-zmr6m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:57:59.382724 3689 pods.go:176] Deleting pod: openshift-console/console-84465d948b-zmr6m I0424 02:57:59.383807 3689 pods.go:236] Attempting to release IPs for pod: openshift-console/console-84465d948b-zmr6m, ips: 10.129.0.12 I0424 02:57:59.383836 3689 obj_retry.go:498] Detected object openshift-console/console-84465d948b-zmr6m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:57:59.393713 3689 base_network_controller_pods.go:487] [default/openshift-console/console-c6f4894b-8h8wn] creating logical port openshift-console_console-c6f4894b-8h8wn for pod on switch ip-10-0-0-104.ec2.internal I0424 02:57:59.393803 3689 kube.go:256] Updating pod openshift-console/console-c6f4894b-8h8wn I0424 02:57:59.407740 3689 pod.go:62] [openshift-console/console-c6f4894b-8h8wn] pod update took 13.960334ms I0424 02:57:59.407760 3689 base_network_controller_pods.go:951] [default/openshift-console/console-c6f4894b-8h8wn] addLogicalPort annotation time took 13.983834ms I0424 02:57:59.408845 3689 pods.go:271] [openshift-console/console-c6f4894b-8h8wn] addLogicalPort took 15.146049ms, libovsdb time 777.591µs I0424 02:57:59.796370 3689 cni.go:452] [openshift-console/console-c6f4894b-8h8wn 83c27dc8ad7d66e5d7e1998b5ddef57cdfc8f9ccffcc36f73b1a2de58313db2e network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-c6f4894b-8h8wn 83c27dc8ad7d66e5d7e1998b5ddef57cdfc8f9ccffcc36f73b1a2de58313db2e network default NAD default NAD key ] I0424 02:57:59.805403 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-c6f4894b-8h8wn, hostIfaceName: 83c27dc8ad7d66e, network: default, NAD default, SandboxID: "83c27dc8ad7d66e5d7e1998b5ddef57cdfc8f9ccffcc36f73b1a2de58313db2e", PCI device ID: , UID: "db5849d4-ed9d-4be2-bed9-3ebfe526ec19", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0424 02:57:59.851587 3689 cni.go:473] [openshift-console/console-c6f4894b-8h8wn 83c27dc8ad7d66e5d7e1998b5ddef57cdfc8f9ccffcc36f73b1a2de58313db2e network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-c6f4894b-8h8wn 83c27dc8ad7d66e5d7e1998b5ddef57cdfc8f9ccffcc36f73b1a2de58313db2e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:ff:77:51:31:4b\",\"name\":\"83c27dc8ad7d66e\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c0087dde-eeae-433c-866c-0ab5f60751bf\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:57:59.956468 3689 obj_retry.go:492] Detected object openshift-console/console-84465d948b-zmr6m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:57:59.956494 3689 obj_retry.go:492] Detected object openshift-console/console-84465d948b-zmr6m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:57:59.960329 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84465d948b-zmr6m I0424 02:57:59.960350 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84465d948b-zmr6m W0424 02:58:06.771005 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0424 02:58:06.771027 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server I0424 02:58:07.014738 3689 obj_retry.go:498] Detected object openshift-console/console-86dc6b9fd4-tgncp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:07.015246 3689 obj_retry.go:498] Detected object openshift-console/console-86dc6b9fd4-tgncp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:07.889903 3689 obj_retry.go:492] Detected object openshift-console/console-86dc6b9fd4-tgncp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:07.889930 3689 obj_retry.go:492] Detected object openshift-console/console-86dc6b9fd4-tgncp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:07.903402 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86dc6b9fd4-tgncp I0424 02:58:07.903418 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86dc6b9fd4-tgncp W0424 02:58:08.268373 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 02:58:08.268398 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 02:58:08.268404 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 02:58:09.918880 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0424 02:58:10.022821 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0424 02:58:18.906580 3689 cni.go:452] [openshift-image-registry/image-registry-64d9447c48-rwxvb 7d9fbc378c889b2f616d8e308fc0a97a70e856cd1bb0358d299ef0b9d0f59fac network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-64d9447c48-rwxvb 7d9fbc378c889b2f616d8e308fc0a97a70e856cd1bb0358d299ef0b9d0f59fac network default NAD default NAD key ] I0424 02:58:18.979605 3689 cni.go:473] [openshift-image-registry/image-registry-64d9447c48-rwxvb 7d9fbc378c889b2f616d8e308fc0a97a70e856cd1bb0358d299ef0b9d0f59fac network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-64d9447c48-rwxvb 7d9fbc378c889b2f616d8e308fc0a97a70e856cd1bb0358d299ef0b9d0f59fac network default NAD default NAD key default], result "{}", err I0424 02:58:19.048915 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-64d9447c48-rwxvb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:19.049016 3689 pods.go:176] Deleting pod: openshift-image-registry/image-registry-64d9447c48-rwxvb I0424 02:58:19.050429 3689 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-64d9447c48-rwxvb, ips: 10.129.0.6 I0424 02:58:19.050458 3689 obj_retry.go:498] Detected object openshift-image-registry/image-registry-64d9447c48-rwxvb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:20.059114 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-64d9447c48-rwxvb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:20.059137 3689 obj_retry.go:492] Detected object openshift-image-registry/image-registry-64d9447c48-rwxvb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:20.068015 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-64d9447c48-rwxvb I0424 02:58:20.068049 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-64d9447c48-rwxvb W0424 02:58:26.185093 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0424 02:58:26.218574 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0424 02:58:27.189572 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0424 02:58:30.376566 3689 obj_retry.go:498] Detected object openshift-ingress/router-default-7bf489c4d9-bkqtb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:30.377168 3689 obj_retry.go:498] Detected object openshift-ingress/router-default-7bf489c4d9-bkqtb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:31.205152 3689 obj_retry.go:492] Detected object openshift-ingress/router-default-7bf489c4d9-bkqtb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:31.205176 3689 obj_retry.go:492] Detected object openshift-ingress/router-default-7bf489c4d9-bkqtb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:31.213146 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7bf489c4d9-bkqtb I0424 02:58:31.213176 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7bf489c4d9-bkqtb W0424 02:58:32.004643 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 02:58:32.004661 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 02:58:32.004673 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 02:58:32.004678 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0424 02:58:35.390983 3689 obj_retry.go:498] Detected object openshift-console/console-84465d948b-lvqcr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:35.391550 3689 obj_retry.go:498] Detected object openshift-console/console-84465d948b-lvqcr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:36.232666 3689 obj_retry.go:492] Detected object openshift-console/console-84465d948b-lvqcr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:36.232693 3689 obj_retry.go:492] Detected object openshift-console/console-84465d948b-lvqcr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:36.239432 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84465d948b-lvqcr I0424 02:58:36.239457 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84465d948b-lvqcr W0424 02:58:43.182826 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 02:58:43.182851 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0424 02:58:50.167704 3689 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653906hm2x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:50.168311 3689 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653906hm2x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:50.168348 3689 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653906hm2x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 02:58:50.168359 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0424 02:58:50.168373 3689 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653906hm2x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 02:58:50.168378 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0424 02:58:51.194249 3689 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653906hm2x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:51.194274 3689 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653906hm2x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:51.194295 3689 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653906hm2x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:51.194311 3689 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653906hm2x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:54.512155 3689 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0424 02:58:54.526460 3689 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0424 02:58:58.176104 3689 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:58.176737 3689 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:58:59.081505 3689 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:59.081554 3689 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:58:59.089551 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0424 02:58:59.089581 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0424 02:59:00.990897 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-86d685cc57-7tt5g] creating logical port openshift-monitoring_telemeter-client-86d685cc57-7tt5g for pod on switch ip-10-0-0-104.ec2.internal I0424 02:59:00.991025 3689 kube.go:256] Updating pod openshift-monitoring/telemeter-client-86d685cc57-7tt5g I0424 02:59:01.016232 3689 pod.go:62] [openshift-monitoring/telemeter-client-86d685cc57-7tt5g] pod update took 25.235717ms I0424 02:59:01.016253 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-86d685cc57-7tt5g] addLogicalPort annotation time took 25.263827ms I0424 02:59:01.032345 3689 pods.go:271] [openshift-monitoring/telemeter-client-86d685cc57-7tt5g] addLogicalPort took 41.417831ms, libovsdb time 12.986388ms I0424 02:59:01.357128 3689 cni.go:452] [openshift-monitoring/telemeter-client-86d685cc57-7tt5g 17d8c129612bedbe7c9a1aabee91b267df3b88378eb69ae551a5b6c588ea4902 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-86d685cc57-7tt5g 17d8c129612bedbe7c9a1aabee91b267df3b88378eb69ae551a5b6c588ea4902 network default NAD default NAD key ] I0424 02:59:01.367929 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-86d685cc57-7tt5g, hostIfaceName: 17d8c129612bedb, network: default, NAD default, SandboxID: "17d8c129612bedbe7c9a1aabee91b267df3b88378eb69ae551a5b6c588ea4902", PCI device ID: , UID: "2b20e7fc-c572-4675-a8da-df47810e280b", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0424 02:59:01.417396 3689 cni.go:473] [openshift-monitoring/telemeter-client-86d685cc57-7tt5g 17d8c129612bedbe7c9a1aabee91b267df3b88378eb69ae551a5b6c588ea4902 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-86d685cc57-7tt5g 17d8c129612bedbe7c9a1aabee91b267df3b88378eb69ae551a5b6c588ea4902 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:00:99:5b:b3:93\",\"name\":\"17d8c129612bedb\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/da5d9d7e-f923-4fe4-bd62-1f55edadfa9c\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:59:02.470577 3689 cni.go:452] [openshift-monitoring/prometheus-k8s-1 6c877a84a908033dfdc06d0206dd847230e55b52b2b51f8e97509a50101d6e10 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 6c877a84a908033dfdc06d0206dd847230e55b52b2b51f8e97509a50101d6e10 network default NAD default NAD key ] I0424 02:59:02.548332 3689 cni.go:473] [openshift-monitoring/prometheus-k8s-1 6c877a84a908033dfdc06d0206dd847230e55b52b2b51f8e97509a50101d6e10 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 6c877a84a908033dfdc06d0206dd847230e55b52b2b51f8e97509a50101d6e10 network default NAD default NAD key default], result "{}", err I0424 02:59:02.614178 3689 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:59:02.614239 3689 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0424 02:59:02.615420 3689 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.129.0.18 I0424 02:59:02.615448 3689 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:59:03.238054 3689 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:59:03.238080 3689 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:59:03.246110 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0424 02:59:03.246134 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0424 02:59:03.293529 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-0-104.ec2.internal I0424 02:59:03.293681 3689 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0424 02:59:03.317231 3689 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 23.588634ms I0424 02:59:03.317257 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 23.616542ms I0424 02:59:03.318597 3689 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 25.097929ms, libovsdb time 994.088µs I0424 02:59:03.862537 3689 cni.go:452] [openshift-monitoring/prometheus-k8s-1 c957897417104fc9060e2942a2bf6161aec1d91ecd2f0901aa2d7291034abce1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 c957897417104fc9060e2942a2bf6161aec1d91ecd2f0901aa2d7291034abce1 network default NAD default NAD key ] I0424 02:59:03.872957 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: c957897417104fc, network: default, NAD default, SandboxID: "c957897417104fc9060e2942a2bf6161aec1d91ecd2f0901aa2d7291034abce1", PCI device ID: , UID: "4569d22e-74e4-4a85-8924-7376c4914464", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0424 02:59:03.918330 3689 cni.go:473] [openshift-monitoring/prometheus-k8s-1 c957897417104fc9060e2942a2bf6161aec1d91ecd2f0901aa2d7291034abce1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 c957897417104fc9060e2942a2bf6161aec1d91ecd2f0901aa2d7291034abce1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:13:3a:ec:b4:af\",\"name\":\"c957897417104fc\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f28c5694-16c3-472e-8542-e92d186873d3\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:59:29.801233 3689 cni.go:452] [openshift-console/console-c6f4894b-8h8wn 83c27dc8ad7d66e5d7e1998b5ddef57cdfc8f9ccffcc36f73b1a2de58313db2e network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-c6f4894b-8h8wn 83c27dc8ad7d66e5d7e1998b5ddef57cdfc8f9ccffcc36f73b1a2de58313db2e network default NAD default NAD key ] I0424 02:59:29.870230 3689 cni.go:473] [openshift-console/console-c6f4894b-8h8wn 83c27dc8ad7d66e5d7e1998b5ddef57cdfc8f9ccffcc36f73b1a2de58313db2e network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-c6f4894b-8h8wn 83c27dc8ad7d66e5d7e1998b5ddef57cdfc8f9ccffcc36f73b1a2de58313db2e network default NAD default NAD key default], result "{}", err I0424 02:59:29.923108 3689 obj_retry.go:498] Detected object openshift-console/console-c6f4894b-8h8wn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:59:29.923143 3689 pods.go:176] Deleting pod: openshift-console/console-c6f4894b-8h8wn I0424 02:59:29.924371 3689 pods.go:236] Attempting to release IPs for pod: openshift-console/console-c6f4894b-8h8wn, ips: 10.129.0.19 I0424 02:59:29.924403 3689 obj_retry.go:498] Detected object openshift-console/console-c6f4894b-8h8wn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:59:29.932839 3689 base_network_controller_pods.go:487] [default/openshift-console/console-5c58bf489-jqrmf] creating logical port openshift-console_console-5c58bf489-jqrmf for pod on switch ip-10-0-0-104.ec2.internal I0424 02:59:29.932933 3689 kube.go:256] Updating pod openshift-console/console-5c58bf489-jqrmf I0424 02:59:29.947109 3689 pod.go:62] [openshift-console/console-5c58bf489-jqrmf] pod update took 14.20528ms I0424 02:59:29.947127 3689 base_network_controller_pods.go:951] [default/openshift-console/console-5c58bf489-jqrmf] addLogicalPort annotation time took 14.226353ms I0424 02:59:29.948177 3689 pods.go:271] [openshift-console/console-5c58bf489-jqrmf] addLogicalPort took 15.354587ms, libovsdb time 744.748µs I0424 02:59:30.292682 3689 cni.go:452] [openshift-console/console-5c58bf489-jqrmf d4af8757af0157699a80fc310c62edf42fe045c44cfa5948238fa6186a1c3011 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-5c58bf489-jqrmf d4af8757af0157699a80fc310c62edf42fe045c44cfa5948238fa6186a1c3011 network default NAD default NAD key ] I0424 02:59:30.302680 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-5c58bf489-jqrmf, hostIfaceName: d4af8757af01576, network: default, NAD default, SandboxID: "d4af8757af0157699a80fc310c62edf42fe045c44cfa5948238fa6186a1c3011", PCI device ID: , UID: "2e5e6c3c-606d-4da1-98c0-99d79a6fc208", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0424 02:59:30.349835 3689 cni.go:473] [openshift-console/console-5c58bf489-jqrmf d4af8757af0157699a80fc310c62edf42fe045c44cfa5948238fa6186a1c3011 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-5c58bf489-jqrmf d4af8757af0157699a80fc310c62edf42fe045c44cfa5948238fa6186a1c3011 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:01:aa:c9:31:bc\",\"name\":\"d4af8757af01576\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90561293-affb-466b-bc1c-0211fe461568\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 02:59:30.352743 3689 obj_retry.go:492] Detected object openshift-console/console-c6f4894b-8h8wn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:59:30.352768 3689 obj_retry.go:492] Detected object openshift-console/console-c6f4894b-8h8wn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:59:30.359082 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-c6f4894b-8h8wn I0424 02:59:30.359108 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-c6f4894b-8h8wn I0424 02:59:30.704815 3689 cni.go:452] [openshift-monitoring/alertmanager-main-0 e277b9c0a825e9abd203abca993ed1bd92d31da3249696de12e20343ef2c15e3 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 e277b9c0a825e9abd203abca993ed1bd92d31da3249696de12e20343ef2c15e3 network default NAD default NAD key ] I0424 02:59:30.777647 3689 cni.go:473] [openshift-monitoring/alertmanager-main-0 e277b9c0a825e9abd203abca993ed1bd92d31da3249696de12e20343ef2c15e3 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 e277b9c0a825e9abd203abca993ed1bd92d31da3249696de12e20343ef2c15e3 network default NAD default NAD key default], result "{}", err I0424 02:59:30.850650 3689 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:59:30.850680 3689 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0424 02:59:30.851878 3689 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.129.0.14 I0424 02:59:30.851905 3689 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:59:31.389855 3689 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:59:31.389884 3689 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:59:31.397787 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0424 02:59:31.397809 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0424 02:59:31.437307 3689 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-104.ec2.internal I0424 02:59:31.437445 3689 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0424 02:59:31.459478 3689 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 22.068386ms I0424 02:59:31.459501 3689 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 22.096478ms I0424 02:59:31.460795 3689 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 23.501814ms, libovsdb time 969.732µs I0424 02:59:31.794253 3689 cni.go:452] [openshift-monitoring/alertmanager-main-0 7534a6a4f6d61dea7669ef1fc28b2998a66fbf0751687ef2e8de15b50176a2f3 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 7534a6a4f6d61dea7669ef1fc28b2998a66fbf0751687ef2e8de15b50176a2f3 network default NAD default NAD key ] I0424 02:59:31.804062 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 7534a6a4f6d61de, network: default, NAD default, SandboxID: "7534a6a4f6d61dea7669ef1fc28b2998a66fbf0751687ef2e8de15b50176a2f3", PCI device ID: , UID: "a1587655-636c-42b6-bd42-2a231f843b24", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0424 02:59:31.848925 3689 cni.go:473] [openshift-monitoring/alertmanager-main-0 7534a6a4f6d61dea7669ef1fc28b2998a66fbf0751687ef2e8de15b50176a2f3 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 7534a6a4f6d61dea7669ef1fc28b2998a66fbf0751687ef2e8de15b50176a2f3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:91:bb:c1:a1:04\",\"name\":\"7534a6a4f6d61de\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d341e48a-9f1a-4755-a049-dba991f437eb\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 02:59:40.416042 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0424 02:59:40.762366 3689 obj_retry.go:498] Detected object openshift-console/console-c6f4894b-bgbdh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 02:59:40.762918 3689 obj_retry.go:498] Detected object openshift-console/console-c6f4894b-bgbdh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 02:59:41.254259 3689 obj_retry.go:492] Detected object openshift-console/console-c6f4894b-bgbdh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:59:41.254295 3689 obj_retry.go:492] Detected object openshift-console/console-c6f4894b-bgbdh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 02:59:41.258251 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-c6f4894b-bgbdh I0424 02:59:41.258275 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-c6f4894b-bgbdh I0424 02:59:55.171780 3689 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0424 03:00:01.864295 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 03:00:01.864313 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 03:00:01.864318 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0424 03:00:04.546913 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0424 03:00:04.546933 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0424 03:00:05.493721 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616660-jhvrk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:00:05.494305 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616660-jhvrk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:00:05.841461 3689 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:00:05.842177 3689 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:00:06.379947 3689 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:00:06.379981 3689 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:00:06.388125 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0424 03:00:06.388144 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0424 03:00:06.515172 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616660-jhvrk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:00:06.515194 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616660-jhvrk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:01:44.124176 3689 cni.go:452] [openshift-console/console-5c58bf489-jqrmf d4af8757af0157699a80fc310c62edf42fe045c44cfa5948238fa6186a1c3011 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-5c58bf489-jqrmf d4af8757af0157699a80fc310c62edf42fe045c44cfa5948238fa6186a1c3011 network default NAD default NAD key ] I0424 03:01:44.202334 3689 cni.go:473] [openshift-console/console-5c58bf489-jqrmf d4af8757af0157699a80fc310c62edf42fe045c44cfa5948238fa6186a1c3011 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-5c58bf489-jqrmf d4af8757af0157699a80fc310c62edf42fe045c44cfa5948238fa6186a1c3011 network default NAD default NAD key default], result "{}", err I0424 03:01:44.252858 3689 obj_retry.go:498] Detected object openshift-console/console-5c58bf489-jqrmf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:01:44.252885 3689 pods.go:176] Deleting pod: openshift-console/console-5c58bf489-jqrmf I0424 03:01:44.254124 3689 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5c58bf489-jqrmf, ips: 10.129.0.22 I0424 03:01:44.254153 3689 obj_retry.go:498] Detected object openshift-console/console-5c58bf489-jqrmf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:01:44.263038 3689 base_network_controller_pods.go:487] [default/openshift-console/console-568d8b9cb4-8lftn] creating logical port openshift-console_console-568d8b9cb4-8lftn for pod on switch ip-10-0-0-104.ec2.internal I0424 03:01:44.263135 3689 kube.go:256] Updating pod openshift-console/console-568d8b9cb4-8lftn I0424 03:01:44.280166 3689 pod.go:62] [openshift-console/console-568d8b9cb4-8lftn] pod update took 17.055047ms I0424 03:01:44.280188 3689 base_network_controller_pods.go:951] [default/openshift-console/console-568d8b9cb4-8lftn] addLogicalPort annotation time took 17.078584ms I0424 03:01:44.281279 3689 pods.go:271] [openshift-console/console-568d8b9cb4-8lftn] addLogicalPort took 18.248905ms, libovsdb time 723.694µs I0424 03:01:44.618399 3689 cni.go:452] [openshift-console/console-568d8b9cb4-8lftn 89c32a450cb03621a3b5fe28da357255f7809ce75b8f26f2717ddbcf058ec410 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-568d8b9cb4-8lftn 89c32a450cb03621a3b5fe28da357255f7809ce75b8f26f2717ddbcf058ec410 network default NAD default NAD key ] I0424 03:01:44.627592 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-568d8b9cb4-8lftn, hostIfaceName: 89c32a450cb0362, network: default, NAD default, SandboxID: "89c32a450cb03621a3b5fe28da357255f7809ce75b8f26f2717ddbcf058ec410", PCI device ID: , UID: "6c28ff12-aa11-4d18-9d49-32fb894f0d58", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0424 03:01:44.673934 3689 cni.go:473] [openshift-console/console-568d8b9cb4-8lftn 89c32a450cb03621a3b5fe28da357255f7809ce75b8f26f2717ddbcf058ec410 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-568d8b9cb4-8lftn 89c32a450cb03621a3b5fe28da357255f7809ce75b8f26f2717ddbcf058ec410 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:a4:a0:e6:58:30\",\"name\":\"89c32a450cb0362\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a9480763-612e-4e2e-8524-d2976769cb07\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:01:44.928029 3689 obj_retry.go:492] Detected object openshift-console/console-5c58bf489-jqrmf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:01:44.928050 3689 obj_retry.go:492] Detected object openshift-console/console-5c58bf489-jqrmf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:01:44.935916 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5c58bf489-jqrmf I0424 03:01:44.935941 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5c58bf489-jqrmf W0424 03:01:54.964492 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0424 03:01:55.043067 3689 obj_retry.go:498] Detected object openshift-console/console-5c58bf489-sxrq2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:01:55.043604 3689 obj_retry.go:498] Detected object openshift-console/console-5c58bf489-sxrq2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:01:55.984553 3689 obj_retry.go:492] Detected object openshift-console/console-5c58bf489-sxrq2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:01:55.984582 3689 obj_retry.go:492] Detected object openshift-console/console-5c58bf489-sxrq2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:01:55.988181 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5c58bf489-sxrq2 I0424 03:01:55.988214 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5c58bf489-sxrq2 I0424 03:02:51.611861 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0424 03:02:52.738004 3689 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0424 03:03:02.515329 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=127 I0424 03:03:13.518274 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0424 03:03:26.612788 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 W0424 03:03:59.446044 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0424 03:04:10.493736 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0424 03:04:10.493794 3689 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-dh5vp IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-dh5vp: no pod IPs found I0424 03:04:15.535801 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=530 I0424 03:04:47.743609 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0424 03:04:57.727828 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0424 03:05:04.716583 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0424 03:05:13.517005 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=73 I0424 03:05:57.733337 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0424 03:05:57.743383 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0424 03:06:02.520523 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0424 03:06:19.722936 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0424 03:06:30.342223 3689 namespace.go:98] [tsf] adding namespace I0424 03:06:30.343145 3689 namespace.go:102] [tsf] adding namespace took 895.858µs I0424 03:06:30.367989 3689 namespace.go:142] [tsf] updating namespace I0424 03:06:30.382337 3689 namespace.go:142] [tsf] updating namespace I0424 03:06:30.517008 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0424 03:06:34.366648 3689 namespace.go:98] [tsf-keycloak] adding namespace I0424 03:06:34.367621 3689 namespace.go:102] [tsf-keycloak] adding namespace took 949.427µs I0424 03:06:34.374306 3689 namespace.go:98] [tsf-tas] adding namespace I0424 03:06:34.374348 3689 namespace.go:98] [rhtpa-operator] adding namespace I0424 03:06:34.374456 3689 namespace.go:98] [konflux-operator] adding namespace I0424 03:06:34.375028 3689 namespace.go:102] [tsf-tas] adding namespace took 701.533µs I0424 03:06:34.375577 3689 namespace.go:102] [rhtpa-operator] adding namespace took 1.213966ms I0424 03:06:34.376299 3689 namespace.go:102] [konflux-operator] adding namespace took 1.822998ms I0424 03:06:34.377008 3689 namespace.go:98] [rhbk-operator] adding namespace I0424 03:06:34.377572 3689 namespace.go:102] [rhbk-operator] adding namespace took 547.908µs I0424 03:06:34.383643 3689 namespace.go:98] [tsf-tpa] adding namespace I0424 03:06:34.383683 3689 namespace.go:98] [cert-manager-operator] adding namespace I0424 03:06:34.383798 3689 namespace.go:98] [konflux-ui] adding namespace I0424 03:06:34.384277 3689 namespace.go:102] [tsf-tpa] adding namespace took 621.442µs I0424 03:06:34.384875 3689 namespace.go:102] [cert-manager-operator] adding namespace took 1.179615ms I0424 03:06:34.385358 3689 namespace.go:102] [konflux-ui] adding namespace took 1.545256ms I0424 03:06:34.407250 3689 namespace.go:142] [tsf-keycloak] updating namespace I0424 03:06:34.412309 3689 namespace.go:142] [konflux-operator] updating namespace I0424 03:06:34.421138 3689 namespace.go:142] [tsf-tas] updating namespace I0424 03:06:34.421259 3689 namespace.go:142] [rhbk-operator] updating namespace I0424 03:06:34.421359 3689 namespace.go:142] [rhtpa-operator] updating namespace I0424 03:06:34.428854 3689 namespace.go:142] [tsf-tpa] updating namespace I0424 03:06:34.428872 3689 namespace.go:142] [tsf-keycloak] updating namespace I0424 03:06:34.436313 3689 namespace.go:142] [cert-manager-operator] updating namespace I0424 03:06:34.436380 3689 namespace.go:142] [konflux-ui] updating namespace I0424 03:06:34.451671 3689 namespace.go:142] [tsf-keycloak] updating namespace I0424 03:06:34.476467 3689 namespace.go:142] [rhtpa-operator] updating namespace I0424 03:06:34.496273 3689 namespace.go:142] [rhtpa-operator] updating namespace I0424 03:06:34.508210 3689 namespace.go:142] [konflux-operator] updating namespace I0424 03:06:34.537204 3689 namespace.go:142] [konflux-operator] updating namespace I0424 03:06:34.548210 3689 namespace.go:142] [rhbk-operator] updating namespace I0424 03:06:34.577296 3689 namespace.go:142] [rhbk-operator] updating namespace I0424 03:06:34.583100 3689 namespace.go:142] [tsf-tpa] updating namespace I0424 03:06:34.589416 3689 namespace.go:142] [tsf-tpa] updating namespace I0424 03:06:34.612014 3689 namespace.go:142] [cert-manager-operator] updating namespace I0424 03:06:34.629112 3689 namespace.go:142] [cert-manager-operator] updating namespace I0424 03:06:34.648114 3689 namespace.go:142] [konflux-ui] updating namespace I0424 03:06:34.667140 3689 namespace.go:142] [konflux-ui] updating namespace I0424 03:06:34.676195 3689 namespace.go:142] [tsf-tas] updating namespace I0424 03:06:34.689456 3689 namespace.go:142] [tsf-tas] updating namespace I0424 03:06:35.644750 3689 namespace.go:142] [tsf-tpa] updating namespace I0424 03:06:35.661987 3689 namespace.go:142] [tsf-keycloak] updating namespace I0424 03:06:40.620075 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0424 03:06:43.465666 3689 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ss6q7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:43.466187 3689 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ss6q7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:43.466210 3689 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ss6q7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:43.466218 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 03:06:43.466230 3689 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ss6q7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:43.466237 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 03:06:43.488131 3689 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588qvfp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:43.488495 3689 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588qvfp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:43.488570 3689 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588qvfp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:43.488582 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 03:06:43.488593 3689 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588qvfp of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:43.488597 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 03:06:44.493847 3689 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ss6q7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:44.493873 3689 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ss6q7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:44.493891 3689 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ss6q7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:44.493902 3689 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ss6q7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:44.515779 3689 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588qvfp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:44.515792 3689 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588qvfp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:44.515801 3689 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588qvfp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:44.515808 3689 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01588qvfp of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:51.129963 3689 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b9nnv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:51.130441 3689 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b9nnv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:51.130463 3689 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b9nnv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:51.130473 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 03:06:51.130491 3689 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b9nnv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:51.130501 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 03:06:51.162595 3689 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d4hwfc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:51.163090 3689 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d4hwfc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:51.163115 3689 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d4hwfc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:51.163125 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 03:06:51.163140 3689 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d4hwfc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:51.163148 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 03:06:52.159733 3689 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b9nnv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:52.159757 3689 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b9nnv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:52.159777 3689 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b9nnv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:52.159793 3689 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2b9nnv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:52.176770 3689 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vjqx8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:52.177354 3689 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vjqx8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:52.177381 3689 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vjqx8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:52.177390 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 03:06:52.177406 3689 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vjqx8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:52.177413 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 03:06:52.191024 3689 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d4hwfc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:52.191040 3689 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d4hwfc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:52.191051 3689 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d4hwfc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:52.191065 3689 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d4hwfc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:52.204647 3689 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136rmhns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:52.205138 3689 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136rmhns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:06:52.205168 3689 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136rmhns of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:52.205178 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0424 03:06:52.205192 3689 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136rmhns of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0424 03:06:52.205199 3689 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0424 03:06:52.980260 3689 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-h8m9d] creating logical port tsf-tpa_rhtpa-operator-controller-manager-586cbb496b-h8m9d for pod on switch ip-10-0-0-104.ec2.internal I0424 03:06:52.980441 3689 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-h8m9d I0424 03:06:52.997253 3689 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-h8m9d] pod update took 16.845318ms I0424 03:06:52.997275 3689 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-h8m9d] addLogicalPort annotation time took 16.875417ms I0424 03:06:52.998565 3689 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-h8m9d] addLogicalPort took 18.321019ms, libovsdb time 878.926µs I0424 03:06:53.207927 3689 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vjqx8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:53.207953 3689 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vjqx8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:53.207964 3689 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vjqx8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:53.207977 3689 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7vjqx8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:53.235562 3689 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136rmhns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:53.235589 3689 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136rmhns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:53.235603 3689 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136rmhns of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:53.235622 3689 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136rmhns of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:06:53.346113 3689 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-h8m9d 99d9e2e3d22da96707e871c4746d1613f546649262f525e1cfcdc319876e263a network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-h8m9d 99d9e2e3d22da96707e871c4746d1613f546649262f525e1cfcdc319876e263a network default NAD default NAD key ] I0424 03:06:53.356041 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-586cbb496b-h8m9d, hostIfaceName: 99d9e2e3d22da96, network: default, NAD default, SandboxID: "99d9e2e3d22da96707e871c4746d1613f546649262f525e1cfcdc319876e263a", PCI device ID: , UID: "3fae8b78-d5d1-4b1c-ba12-75fde5aff124", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0424 03:06:53.405152 3689 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-h8m9d 99d9e2e3d22da96707e871c4746d1613f546649262f525e1cfcdc319876e263a network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-h8m9d 99d9e2e3d22da96707e871c4746d1613f546649262f525e1cfcdc319876e263a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:5e:d7:2f:ee:8a\",\"name\":\"99d9e2e3d22da96\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2f56e144-1e2a-4c47-9962-5c0c18c14a14\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 03:06:56.154326 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0424 03:06:57.701383 3689 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9] creating logical port konflux-operator_konflux-operator-controller-manager-66b879f574-t25x9 for pod on switch ip-10-0-0-104.ec2.internal I0424 03:06:57.701532 3689 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9 I0424 03:06:57.755036 3689 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9 I0424 03:06:57.791393 3689 pod.go:62] [konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9] pod update took 89.917348ms I0424 03:06:57.791413 3689 base_network_controller_pods.go:951] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9] addLogicalPort annotation time took 89.941655ms I0424 03:06:57.792682 3689 pods.go:271] [konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9] addLogicalPort took 91.316274ms, libovsdb time 881.482µs I0424 03:06:58.037202 3689 cni.go:452] [konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9 924bf15c578cc3e2a14189f51715b6165e5fe67eadbda8c4be8edad818d18674 network default NAD default NAD key ] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9 924bf15c578cc3e2a14189f51715b6165e5fe67eadbda8c4be8edad818d18674 network default NAD default NAD key ] I0424 03:06:58.047707 3689 helper_linux.go:556] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-66b879f574-t25x9, hostIfaceName: 924bf15c578cc3e, network: default, NAD default, SandboxID: "924bf15c578cc3e2a14189f51715b6165e5fe67eadbda8c4be8edad818d18674", PCI device ID: , UID: "bcf3f63b-e036-4801-ae7c-831b33b8297d", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0424 03:06:58.094141 3689 cni.go:473] [konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9 924bf15c578cc3e2a14189f51715b6165e5fe67eadbda8c4be8edad818d18674 network default NAD default NAD key default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-t25x9 924bf15c578cc3e2a14189f51715b6165e5fe67eadbda8c4be8edad818d18674 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:05:03:69:8b:0b\",\"name\":\"924bf15c578cc3e\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e2347207-8deb-48ef-9d87-2a1ba4a1e414\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 03:07:00.417550 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0424 03:07:00.430310 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0424 03:07:00.451800 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0424 03:07:00.463755 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0424 03:07:01.191170 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0424 03:07:07.190758 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0424 03:07:12.196538 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0424 03:07:14.222960 3689 namespace.go:98] [cert-manager] adding namespace I0424 03:07:14.223867 3689 namespace.go:102] [cert-manager] adding namespace took 884.659µs I0424 03:07:14.248696 3689 namespace.go:142] [cert-manager] updating namespace I0424 03:07:14.265875 3689 namespace.go:142] [cert-manager] updating namespace I0424 03:07:14.276175 3689 namespace.go:142] [openshift-operators] updating namespace I0424 03:07:14.300812 3689 namespace.go:142] [openshift-operators] updating namespace I0424 03:07:14.527672 3689 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-jlh9n] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-jlh9n for pod on switch ip-10-0-0-104.ec2.internal I0424 03:07:14.527786 3689 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-jlh9n I0424 03:07:14.563358 3689 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-jlh9n] pod update took 35.601189ms I0424 03:07:14.563383 3689 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-jlh9n] addLogicalPort annotation time took 35.63391ms I0424 03:07:14.564500 3689 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-jlh9n] addLogicalPort took 36.844587ms, libovsdb time 786.688µs I0424 03:07:14.889478 3689 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-jlh9n c38da287f715359f7497cfbfb09613a7626d3b219461f0d9f86473d4c4a300a6 network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-jlh9n c38da287f715359f7497cfbfb09613a7626d3b219461f0d9f86473d4c4a300a6 network default NAD default NAD key ] I0424 03:07:14.899015 3689 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-jlh9n, hostIfaceName: c38da287f715359, network: default, NAD default, SandboxID: "c38da287f715359f7497cfbfb09613a7626d3b219461f0d9f86473d4c4a300a6", PCI device ID: , UID: "7ac31e7b-a8e2-4ad6-bd7b-1a779617ca50", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0424 03:07:14.945728 3689 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-jlh9n c38da287f715359f7497cfbfb09613a7626d3b219461f0d9f86473d4c4a300a6 network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-jlh9n c38da287f715359f7497cfbfb09613a7626d3b219461f0d9f86473d4c4a300a6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:ac:17:e7:40:52\",\"name\":\"c38da287f715359\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6206d0b2-f6d6-443b-a3ca-41d5f8de652e\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 03:07:19.211867 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0424 03:07:19.211886 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0424 03:07:21.031203 3689 namespace.go:142] [openshift-operators] updating namespace I0424 03:07:21.598595 3689 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-nqc4r] creating logical port openshift-operators_openshift-pipelines-operator-6bdf6c8794-nqc4r for pod on switch ip-10-0-0-104.ec2.internal I0424 03:07:21.598713 3689 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-nqc4r I0424 03:07:21.624532 3689 pod.go:62] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-nqc4r] pod update took 25.846222ms I0424 03:07:21.624554 3689 base_network_controller_pods.go:951] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-nqc4r] addLogicalPort annotation time took 25.872198ms I0424 03:07:21.625726 3689 pods.go:271] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-nqc4r] addLogicalPort took 27.150783ms, libovsdb time 866.793µs I0424 03:07:21.993306 3689 cni.go:452] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-nqc4r c07971f173f4c7aec065171741f554fe0455ed5349ca1b872534819ab7366a91 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-nqc4r c07971f173f4c7aec065171741f554fe0455ed5349ca1b872534819ab7366a91 network default NAD default NAD key ] I0424 03:07:22.003242 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6bdf6c8794-nqc4r, hostIfaceName: c07971f173f4c7a, network: default, NAD default, SandboxID: "c07971f173f4c7aec065171741f554fe0455ed5349ca1b872534819ab7366a91", PCI device ID: , UID: "1fc1e155-1c97-4e68-b2b5-8c349a27c025", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0424 03:07:22.050954 3689 cni.go:473] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-nqc4r c07971f173f4c7aec065171741f554fe0455ed5349ca1b872534819ab7366a91 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-nqc4r c07971f173f4c7aec065171741f554fe0455ed5349ca1b872534819ab7366a91 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:2e:58:ba:ab:7c\",\"name\":\"c07971f173f4c7a\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fc96b9c6-0440-4c03-ba04-e861e6eab0c9\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:07:28.747303 3689 namespace.go:98] [trusted-artifact-signer] adding namespace I0424 03:07:28.748178 3689 namespace.go:102] [trusted-artifact-signer] adding namespace took 848.698µs I0424 03:07:28.780164 3689 namespace.go:142] [trusted-artifact-signer] updating namespace I0424 03:07:28.798724 3689 namespace.go:142] [trusted-artifact-signer] updating namespace W0424 03:07:29.306094 3689 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0424 03:07:36.816022 3689 namespace.go:98] [openshift-pipelines] adding namespace I0424 03:07:36.816884 3689 namespace.go:102] [openshift-pipelines] adding namespace took 841.55µs I0424 03:07:36.836306 3689 namespace.go:142] [openshift-pipelines] updating namespace I0424 03:07:37.814839 3689 namespace.go:142] [cert-manager] updating namespace I0424 03:07:38.011793 3689 namespace.go:142] [cert-manager-operator] updating namespace I0424 03:07:38.415252 3689 namespace.go:142] [konflux-operator] updating namespace I0424 03:07:38.617485 3689 namespace.go:142] [konflux-ui] updating namespace I0424 03:07:38.814608 3689 namespace.go:142] [open-cluster-management-2ps8kd9n54vkadsmhv5dl8r11de9333h] updating namespace I0424 03:07:39.216353 3689 namespace.go:142] [rhbk-operator] updating namespace I0424 03:07:39.420247 3689 namespace.go:142] [rhtpa-operator] updating namespace I0424 03:07:39.616909 3689 namespace.go:142] [trusted-artifact-signer] updating namespace I0424 03:07:39.816424 3689 namespace.go:142] [tsf] updating namespace I0424 03:07:40.021544 3689 namespace.go:142] [tsf-keycloak] updating namespace I0424 03:07:40.217574 3689 namespace.go:142] [tsf-tas] updating namespace I0424 03:07:40.424679 3689 namespace.go:142] [tsf-tpa] updating namespace I0424 03:07:40.568068 3689 namespace.go:142] [cert-manager] updating namespace I0424 03:07:40.580799 3689 namespace.go:142] [cert-manager-operator] updating namespace I0424 03:07:40.597356 3689 namespace.go:142] [default] updating namespace I0424 03:07:40.611072 3689 namespace.go:142] [konflux-operator] updating namespace I0424 03:07:40.625295 3689 namespace.go:142] [konflux-ui] updating namespace I0424 03:07:40.655409 3689 namespace.go:142] [open-cluster-management-2ps8kd9n54vkadsmhv5dl8r11de9333h] updating namespace I0424 03:07:40.665240 3689 namespace.go:142] [openshift] updating namespace I0424 03:07:40.682260 3689 namespace.go:142] [rhbk-operator] updating namespace I0424 03:07:40.699211 3689 namespace.go:142] [rhtpa-operator] updating namespace I0424 03:07:40.707189 3689 namespace.go:142] [trusted-artifact-signer] updating namespace I0424 03:07:40.726099 3689 namespace.go:142] [tsf] updating namespace I0424 03:07:40.740688 3689 namespace.go:142] [tsf-keycloak] updating namespace I0424 03:07:40.763250 3689 namespace.go:142] [tsf-tas] updating namespace I0424 03:07:40.787023 3689 namespace.go:142] [tsf-tpa] updating namespace I0424 03:07:40.835007 3689 namespace.go:142] [cert-manager] updating namespace I0424 03:07:40.942426 3689 namespace.go:142] [cert-manager-operator] updating namespace I0424 03:07:41.013145 3689 namespace.go:142] [default] updating namespace I0424 03:07:41.106061 3689 namespace.go:142] [konflux-operator] updating namespace I0424 03:07:41.201869 3689 namespace.go:142] [konflux-ui] updating namespace I0424 03:07:41.302929 3689 namespace.go:142] [open-cluster-management-2ps8kd9n54vkadsmhv5dl8r11de9333h] updating namespace I0424 03:07:41.425727 3689 namespace.go:142] [openshift] updating namespace I0424 03:07:41.522900 3689 namespace.go:142] [rhbk-operator] updating namespace I0424 03:07:41.637276 3689 namespace.go:142] [rhtpa-operator] updating namespace I0424 03:07:41.721196 3689 namespace.go:142] [trusted-artifact-signer] updating namespace I0424 03:07:41.901690 3689 namespace.go:142] [tsf] updating namespace I0424 03:07:42.045275 3689 namespace.go:142] [tsf-keycloak] updating namespace I0424 03:07:42.208814 3689 namespace.go:142] [tsf-tas] updating namespace I0424 03:07:42.324819 3689 namespace.go:142] [tsf-tpa] updating namespace I0424 03:07:42.431675 3689 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:07:42.432277 3689 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:07:42.470499 3689 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:07:42.470537 3689 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:07:42.478445 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0424 03:07:42.478465 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0424 03:07:42.859639 3689 namespace.go:142] [openshift-pipelines] updating namespace I0424 03:07:51.050762 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q for pod on switch ip-10-0-0-104.ec2.internal I0424 03:07:51.050862 3689 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q I0424 03:07:51.066300 3689 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q] pod update took 15.468427ms I0424 03:07:51.066382 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q] addLogicalPort annotation time took 15.553811ms I0424 03:07:51.068120 3689 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q] addLogicalPort took 17.375608ms, libovsdb time 1.246718ms I0424 03:07:51.125821 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-rkm4q] creating logical port openshift-pipelines_tekton-pipelines-webhook-7b6d6f4c94-rkm4q for pod on switch ip-10-0-0-104.ec2.internal I0424 03:07:51.125945 3689 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-rkm4q I0424 03:07:51.143741 3689 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-rkm4q] pod update took 17.83457ms I0424 03:07:51.143765 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-rkm4q] addLogicalPort annotation time took 17.865653ms I0424 03:07:51.144984 3689 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-rkm4q] addLogicalPort took 19.183274ms, libovsdb time 844.439µs I0424 03:07:51.243829 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-qdd9d] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-57f84cff6f-qdd9d for pod on switch ip-10-0-0-104.ec2.internal I0424 03:07:51.243931 3689 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-qdd9d I0424 03:07:51.259847 3689 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-qdd9d] pod update took 15.949983ms I0424 03:07:51.259868 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-qdd9d] addLogicalPort annotation time took 15.975977ms I0424 03:07:51.261173 3689 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-qdd9d] addLogicalPort took 17.35389ms, libovsdb time 929.966µs I0424 03:07:51.421556 3689 cni.go:452] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q c29eccdb51c422d4140646f3ce85bce146185cfe0b38238fe82f2ba7c544add1 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q c29eccdb51c422d4140646f3ce85bce146185cfe0b38238fe82f2ba7c544add1 network default NAD default NAD key ] I0424 03:07:51.437978 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q, hostIfaceName: c29eccdb51c422d, network: default, NAD default, SandboxID: "c29eccdb51c422d4140646f3ce85bce146185cfe0b38238fe82f2ba7c544add1", PCI device ID: , UID: "dbc6787d-bfcb-48fa-ae22-34e6e1db3be0", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0424 03:07:51.494639 3689 cni.go:473] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q c29eccdb51c422d4140646f3ce85bce146185cfe0b38238fe82f2ba7c544add1 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-7zh8q c29eccdb51c422d4140646f3ce85bce146185cfe0b38238fe82f2ba7c544add1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:81:d6:1e:fd:72\",\"name\":\"c29eccdb51c422d\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b855baf9-359e-4e06-9ce2-5a2b8b6d33af\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:07:51.497305 3689 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-rkm4q 2f31582002ef020d8ce4a0f512d0391f2a056166824dfe0c267ebfb4843c5bff network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-rkm4q 2f31582002ef020d8ce4a0f512d0391f2a056166824dfe0c267ebfb4843c5bff network default NAD default NAD key ] I0424 03:07:51.506631 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-7b6d6f4c94-rkm4q, hostIfaceName: 2f31582002ef020, network: default, NAD default, SandboxID: "2f31582002ef020d8ce4a0f512d0391f2a056166824dfe0c267ebfb4843c5bff", PCI device ID: , UID: "ee2fff9b-ee24-4610-bbb3-f157ed87fe72", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0424 03:07:51.555060 3689 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-rkm4q 2f31582002ef020d8ce4a0f512d0391f2a056166824dfe0c267ebfb4843c5bff network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-rkm4q 2f31582002ef020d8ce4a0f512d0391f2a056166824dfe0c267ebfb4843c5bff network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:e1:5f:3f:31:20\",\"name\":\"2f31582002ef020\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8f154aa8-2f45-41bc-b05d-9e0a893af7a9\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:07:51.605520 3689 cni.go:452] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-qdd9d 72e5d80e5fbd5a1857e4aa0e2c8ee7ab473f1f1ce19f94d7f4f652eb28e86db9 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-qdd9d 72e5d80e5fbd5a1857e4aa0e2c8ee7ab473f1f1ce19f94d7f4f652eb28e86db9 network default NAD default NAD key ] I0424 03:07:51.615585 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-57f84cff6f-qdd9d, hostIfaceName: 72e5d80e5fbd5a1, network: default, NAD default, SandboxID: "72e5d80e5fbd5a1857e4aa0e2c8ee7ab473f1f1ce19f94d7f4f652eb28e86db9", PCI device ID: , UID: "369456fa-f054-4388-97e6-8efdd0da267a", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0424 03:07:51.661719 3689 cni.go:473] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-qdd9d 72e5d80e5fbd5a1857e4aa0e2c8ee7ab473f1f1ce19f94d7f4f652eb28e86db9 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-qdd9d 72e5d80e5fbd5a1857e4aa0e2c8ee7ab473f1f1ce19f94d7f4f652eb28e86db9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:54:5d:82:78:80\",\"name\":\"72e5d80e5fbd5a1\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/20fa167b-44e9-449e-a814-a60b651d53c0\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 03:07:54.357067 3689 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0424 03:07:54.391493 3689 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0424 03:07:54.391658 3689 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0424 03:07:54.391675 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0424 03:07:54.391679 3689 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0424 03:07:55.387848 3689 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0424 03:07:55.387865 3689 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0424 03:07:55.387872 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0424 03:07:57.517215 3689 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 03:07:57.517230 3689 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 03:07:57.517234 3689 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 03:07:57.517238 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 03:08:08.462554 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 03:08:08.462579 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 03:08:08.462586 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0424 03:08:08.462593 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0424 03:08:15.007405 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-s2qg6] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-s2qg6 for pod on switch ip-10-0-0-104.ec2.internal I0424 03:08:15.007490 3689 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-s2qg6 I0424 03:08:15.021302 3689 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-s2qg6] pod update took 13.837295ms I0424 03:08:15.021321 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-s2qg6] addLogicalPort annotation time took 13.857197ms I0424 03:08:15.022573 3689 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-s2qg6] addLogicalPort took 15.184648ms, libovsdb time 855.933µs I0424 03:08:15.107344 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-rrdrn] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-rrdrn for pod on switch ip-10-0-0-104.ec2.internal I0424 03:08:15.107437 3689 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-rrdrn I0424 03:08:15.120741 3689 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-rrdrn] pod update took 13.322329ms I0424 03:08:15.120759 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-rrdrn] addLogicalPort annotation time took 13.344318ms I0424 03:08:15.121929 3689 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-rrdrn] addLogicalPort took 14.595126ms, libovsdb time 856.55µs I0424 03:08:15.368033 3689 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-s2qg6 0c78575898cda1c49267f109954ff0f30b18f944dcc27a4f1ec7c788ee733277 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-s2qg6 0c78575898cda1c49267f109954ff0f30b18f944dcc27a4f1ec7c788ee733277 network default NAD default NAD key ] I0424 03:08:15.377774 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-s2qg6, hostIfaceName: 0c78575898cda1c, network: default, NAD default, SandboxID: "0c78575898cda1c49267f109954ff0f30b18f944dcc27a4f1ec7c788ee733277", PCI device ID: , UID: "32c9e893-5c99-4f33-b593-16da3e78da26", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0424 03:08:15.424878 3689 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-s2qg6 0c78575898cda1c49267f109954ff0f30b18f944dcc27a4f1ec7c788ee733277 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-s2qg6 0c78575898cda1c49267f109954ff0f30b18f944dcc27a4f1ec7c788ee733277 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:a0:33:68:c1:ef\",\"name\":\"0c78575898cda1c\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/16be023d-694f-4d2d-896d-270708e5d461\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:08:15.467103 3689 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-rrdrn a5928628b7a67a44bd0c5845ed1cc44b2b7bc21ca59cc43706c7425415548d10 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-rrdrn a5928628b7a67a44bd0c5845ed1cc44b2b7bc21ca59cc43706c7425415548d10 network default NAD default NAD key ] I0424 03:08:15.476896 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-rrdrn, hostIfaceName: a5928628b7a67a4, network: default, NAD default, SandboxID: "a5928628b7a67a44bd0c5845ed1cc44b2b7bc21ca59cc43706c7425415548d10", PCI device ID: , UID: "2c262ab4-dba0-40d1-98b2-c99ff815b0f9", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0424 03:08:15.523095 3689 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-rrdrn a5928628b7a67a44bd0c5845ed1cc44b2b7bc21ca59cc43706c7425415548d10 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-rrdrn a5928628b7a67a44bd0c5845ed1cc44b2b7bc21ca59cc43706c7425415548d10 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:8f:8d:c0:00:c8\",\"name\":\"a5928628b7a67a4\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba351b72-41ce-4604-a578-bfaf367a1295\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:08:20.622156 3689 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:08:20.622701 3689 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:08:20.643279 3689 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:08:20.643297 3689 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:08:20.650608 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0424 03:08:20.650627 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0424 03:08:22.613716 3689 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0424 03:08:22.614356 3689 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0424 03:08:22.614372 3689 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0424 03:08:22.616818 3689 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0424 03:08:22.636651 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0424 03:08:32.520610 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=45 W0424 03:08:33.590973 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0424 03:08:35.594299 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0424 03:08:35.594320 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0424 03:08:35.594435 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0424 03:08:35.594445 3689 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0424 03:08:39.561607 3689 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0424 03:08:42.651820 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-0-104.ec2.internal I0424 03:08:42.651907 3689 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0424 03:08:42.663825 3689 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 11.939085ms I0424 03:08:42.663845 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 11.96427ms I0424 03:08:42.665227 3689 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 13.430456ms, libovsdb time 1.005873ms I0424 03:08:52.612786 3689 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 2d4e93055fb6f924e308f8d5c7d54801e7d223342b7f5aba35c843b2b6b789fa network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 2d4e93055fb6f924e308f8d5c7d54801e7d223342b7f5aba35c843b2b6b789fa network default NAD default NAD key ] I0424 03:08:52.622984 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 2d4e93055fb6f92, network: default, NAD default, SandboxID: "2d4e93055fb6f924e308f8d5c7d54801e7d223342b7f5aba35c843b2b6b789fa", PCI device ID: , UID: "d0d0c351-bde5-4cbe-a4cc-07a51f5f78b4", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0424 03:08:52.671298 3689 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 2d4e93055fb6f924e308f8d5c7d54801e7d223342b7f5aba35c843b2b6b789fa network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 2d4e93055fb6f924e308f8d5c7d54801e7d223342b7f5aba35c843b2b6b789fa network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:ec:62:42:3e:4a\",\"name\":\"2d4e93055fb6f92\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/89d472ce-26f7-4c71-9f6b-df7b27ee9b87\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:08:57.792386 3689 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0424 03:09:04.719121 3689 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0424 03:09:04.719145 3689 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0424 03:09:04.719152 3689 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0424 03:09:12.998518 3689 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:09:12.999011 3689 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:09:13.037095 3689 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:09:13.037115 3689 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:09:13.043771 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0424 03:09:13.043796 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0424 03:09:14.989212 3689 namespace.go:142] [openshift-pipelines] updating namespace I0424 03:09:15.728913 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-67x2c] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-67x2c for pod on switch ip-10-0-0-104.ec2.internal I0424 03:09:15.729008 3689 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-67x2c I0424 03:09:15.742130 3689 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-67x2c] pod update took 13.149181ms I0424 03:09:15.742153 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/tkn-cli-serve-784b566968-67x2c] addLogicalPort annotation time took 13.175245ms I0424 03:09:15.743400 3689 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-67x2c] addLogicalPort took 14.503577ms, libovsdb time 882.278µs I0424 03:09:16.086130 3689 cni.go:452] [openshift-pipelines/tkn-cli-serve-784b566968-67x2c bf37a475b4c33c98002bb9f54d5fe416eb7004bcee0448cbbf6c0015fbe251e1 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-67x2c bf37a475b4c33c98002bb9f54d5fe416eb7004bcee0448cbbf6c0015fbe251e1 network default NAD default NAD key ] I0424 03:09:16.095836 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-67x2c, hostIfaceName: bf37a475b4c33c9, network: default, NAD default, SandboxID: "bf37a475b4c33c98002bb9f54d5fe416eb7004bcee0448cbbf6c0015fbe251e1", PCI device ID: , UID: "aedf1e0f-53fb-44aa-a7ba-554dfac76e59", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0424 03:09:16.099838 3689 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-t7fg2-jbwgp] creating logical port tsf-tas_ctlog-createtree-job-t7fg2-jbwgp for pod on switch ip-10-0-0-104.ec2.internal I0424 03:09:16.099960 3689 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-t7fg2-jbwgp I0424 03:09:16.125411 3689 pod.go:62] [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp] pod update took 25.48036ms I0424 03:09:16.125434 3689 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-createtree-job-t7fg2-jbwgp] addLogicalPort annotation time took 25.517719ms I0424 03:09:16.127048 3689 pods.go:271] [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp] addLogicalPort took 27.226405ms, libovsdb time 1.052189ms I0424 03:09:16.145082 3689 cni.go:473] [openshift-pipelines/tkn-cli-serve-784b566968-67x2c bf37a475b4c33c98002bb9f54d5fe416eb7004bcee0448cbbf6c0015fbe251e1 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-67x2c bf37a475b4c33c98002bb9f54d5fe416eb7004bcee0448cbbf6c0015fbe251e1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:fe:a6:2f:1a:06\",\"name\":\"bf37a475b4c33c9\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7ea2ddb0-8682-4093-819e-fbea9e60a6fa\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:09:16.158253 3689 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-5b97cfbfbf-p2pw4] creating logical port tsf-tas_trillian-logserver-5b97cfbfbf-p2pw4 for pod on switch ip-10-0-0-104.ec2.internal I0424 03:09:16.158340 3689 kube.go:256] Updating pod tsf-tas/trillian-logserver-5b97cfbfbf-p2pw4 I0424 03:09:16.173559 3689 pod.go:62] [tsf-tas/trillian-logserver-5b97cfbfbf-p2pw4] pod update took 15.238409ms I0424 03:09:16.173585 3689 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logserver-5b97cfbfbf-p2pw4] addLogicalPort annotation time took 15.274645ms I0424 03:09:16.174810 3689 pods.go:271] [tsf-tas/trillian-logserver-5b97cfbfbf-p2pw4] addLogicalPort took 16.568463ms, libovsdb time 784.115µs I0424 03:09:16.519543 3689 cni.go:452] [tsf-tas/trillian-logserver-5b97cfbfbf-p2pw4 8300d60f1a9435226f1f563a9a5259249ad7f5d767661ec0b4a9d4b11b3859f5 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logserver-5b97cfbfbf-p2pw4 8300d60f1a9435226f1f563a9a5259249ad7f5d767661ec0b4a9d4b11b3859f5 network default NAD default NAD key ] I0424 03:09:16.532607 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-5b97cfbfbf-p2pw4, hostIfaceName: 8300d60f1a94352, network: default, NAD default, SandboxID: "8300d60f1a9435226f1f563a9a5259249ad7f5d767661ec0b4a9d4b11b3859f5", PCI device ID: , UID: "1f77e199-a63b-4b60-bc5f-817b55a3c75f", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0424 03:09:16.582781 3689 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-88t6m-8kczv] creating logical port tsf-tas_rekor-createtree-job-88t6m-8kczv for pod on switch ip-10-0-0-104.ec2.internal I0424 03:09:16.582921 3689 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-88t6m-8kczv I0424 03:09:16.585312 3689 cni.go:473] [tsf-tas/trillian-logserver-5b97cfbfbf-p2pw4 8300d60f1a9435226f1f563a9a5259249ad7f5d767661ec0b4a9d4b11b3859f5 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logserver-5b97cfbfbf-p2pw4 8300d60f1a9435226f1f563a9a5259249ad7f5d767661ec0b4a9d4b11b3859f5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:f2:d8:67:27:e1\",\"name\":\"8300d60f1a94352\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b2c4c1ef-81b4-4580-87c8-ab78c45626e8\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:09:16.599645 3689 pod.go:62] [tsf-tas/rekor-createtree-job-88t6m-8kczv] pod update took 16.752565ms I0424 03:09:16.599665 3689 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-88t6m-8kczv] addLogicalPort annotation time took 16.77673ms I0424 03:09:16.600960 3689 pods.go:271] [tsf-tas/rekor-createtree-job-88t6m-8kczv] addLogicalPort took 18.198997ms, libovsdb time 927.362µs I0424 03:09:16.757897 3689 cni.go:452] [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp b91fc1a5d395695cb14718c758e8b145928f891345cf2904ec80ae8b921a95fc network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp b91fc1a5d395695cb14718c758e8b145928f891345cf2904ec80ae8b921a95fc network default NAD default NAD key ] I0424 03:09:16.767815 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-t7fg2-jbwgp, hostIfaceName: b91fc1a5d395695, network: default, NAD default, SandboxID: "b91fc1a5d395695cb14718c758e8b145928f891345cf2904ec80ae8b921a95fc", PCI device ID: , UID: "7cd83535-f5c7-47e3-a0be-de2ec5a9bb2a", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0424 03:09:16.818049 3689 cni.go:473] [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp b91fc1a5d395695cb14718c758e8b145928f891345cf2904ec80ae8b921a95fc network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp b91fc1a5d395695cb14718c758e8b145928f891345cf2904ec80ae8b921a95fc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:31:c8:2b:57:e9\",\"name\":\"b91fc1a5d395695\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dc58dbd8-79e5-4040-a89c-565a619f0e5a\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:09:16.955133 3689 cni.go:452] [tsf-tas/rekor-createtree-job-88t6m-8kczv a3d14c8012b6902cbcef7d332fd8f0378ca4f5450798dd4253df85dd8d872a48 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-88t6m-8kczv a3d14c8012b6902cbcef7d332fd8f0378ca4f5450798dd4253df85dd8d872a48 network default NAD default NAD key ] I0424 03:09:16.965962 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-88t6m-8kczv, hostIfaceName: a3d14c8012b6902, network: default, NAD default, SandboxID: "a3d14c8012b6902cbcef7d332fd8f0378ca4f5450798dd4253df85dd8d872a48", PCI device ID: , UID: "1bb4893d-1492-45dc-91b3-fb41048974d3", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0424 03:09:17.030016 3689 cni.go:473] [tsf-tas/rekor-createtree-job-88t6m-8kczv a3d14c8012b6902cbcef7d332fd8f0378ca4f5450798dd4253df85dd8d872a48 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-88t6m-8kczv a3d14c8012b6902cbcef7d332fd8f0378ca4f5450798dd4253df85dd8d872a48 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:34:03:7e:0f:05\",\"name\":\"a3d14c8012b6902\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/41d8067d-c476-45cd-92bb-6624eb8d3df2\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 03:09:18.788050 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0424 03:09:18.788074 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0424 03:09:24.895211 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0424 03:09:25.825955 3689 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0424 03:09:29.147865 3689 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-5n7mp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:09:29.148469 3689 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-5n7mp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:09:29.967805 3689 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-5n7mp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:09:29.967827 3689 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-5n7mp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0424 03:09:29.970834 3689 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0424 03:09:33.092703 3689 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-2x7b7-z62nr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:09:33.093225 3689 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-2x7b7-z62nr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:09:33.983940 3689 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-2x7b7-z62nr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:09:33.983964 3689 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-2x7b7-z62nr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0424 03:09:46.044715 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0424 03:09:47.059631 3689 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0424 03:09:47.059649 3689 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0424 03:09:54.001211 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0424 03:09:58.130687 3689 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-hnjk7] creating logical port openshift-pipelines_pipelines-console-plugin-65f788bc74-hnjk7 for pod on switch ip-10-0-0-104.ec2.internal I0424 03:09:58.130768 3689 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-65f788bc74-hnjk7 I0424 03:09:58.150008 3689 pod.go:62] [openshift-pipelines/pipelines-console-plugin-65f788bc74-hnjk7] pod update took 19.264043ms I0424 03:09:58.150032 3689 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-hnjk7] addLogicalPort annotation time took 19.292312ms I0424 03:09:58.151288 3689 pods.go:271] [openshift-pipelines/pipelines-console-plugin-65f788bc74-hnjk7] addLogicalPort took 20.619ms, libovsdb time 874.008µs I0424 03:09:58.685949 3689 cni.go:452] [openshift-pipelines/pipelines-console-plugin-65f788bc74-hnjk7 ae130d151ea312a2b55bf4c06aa9d972e615a8b22682d7d6657c7152df3f6e07 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-hnjk7 ae130d151ea312a2b55bf4c06aa9d972e615a8b22682d7d6657c7152df3f6e07 network default NAD default NAD key ] I0424 03:09:58.694996 3689 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-65f788bc74-hnjk7, hostIfaceName: ae130d151ea312a, network: default, NAD default, SandboxID: "ae130d151ea312a2b55bf4c06aa9d972e615a8b22682d7d6657c7152df3f6e07", PCI device ID: , UID: "689b98d1-6830-4219-bf66-b9b93ca544eb", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0424 03:09:58.749795 3689 cni.go:473] [openshift-pipelines/pipelines-console-plugin-65f788bc74-hnjk7 ae130d151ea312a2b55bf4c06aa9d972e615a8b22682d7d6657c7152df3f6e07 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-hnjk7 ae130d151ea312a2b55bf4c06aa9d972e615a8b22682d7d6657c7152df3f6e07 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:74:42:5d:c8:90\",\"name\":\"ae130d151ea312a\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7a8996af-f629-4198-9712-7e72f1e27f9a\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:10:02.368385 3689 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:02.368930 3689 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:02.388414 3689 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:02.388437 3689 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:02.395663 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0424 03:10:02.395678 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0424 03:10:05.331775 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616670-7gkpd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:05.332332 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616670-7gkpd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:06.103554 3689 cni.go:452] [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp b91fc1a5d395695cb14718c758e8b145928f891345cf2904ec80ae8b921a95fc network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp b91fc1a5d395695cb14718c758e8b145928f891345cf2904ec80ae8b921a95fc network default NAD default NAD key ] I0424 03:10:06.190570 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616670-7gkpd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:06.190591 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616670-7gkpd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:06.190903 3689 cni.go:473] [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp b91fc1a5d395695cb14718c758e8b145928f891345cf2904ec80ae8b921a95fc network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-t7fg2-jbwgp b91fc1a5d395695cb14718c758e8b145928f891345cf2904ec80ae8b921a95fc network default NAD default NAD key default], result "{}", err I0424 03:10:06.242279 3689 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-t7fg2-jbwgp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:06.242307 3689 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-t7fg2-jbwgp I0424 03:10:06.243587 3689 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-t7fg2-jbwgp, ips: 10.129.0.36 I0424 03:10:06.243613 3689 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-t7fg2-jbwgp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:07.135205 3689 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-t7fg2-jbwgp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:07.135231 3689 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-t7fg2-jbwgp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:11.187605 3689 obj_retry.go:498] Detected object tsf/patch-tekton-config-rhkbk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:11.188191 3689 obj_retry.go:498] Detected object tsf/patch-tekton-config-rhkbk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:12.076285 3689 obj_retry.go:492] Detected object tsf/patch-tekton-config-rhkbk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:12.076311 3689 obj_retry.go:492] Detected object tsf/patch-tekton-config-rhkbk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:12.196285 3689 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-h8m6v] creating logical port tsf_tsf-tekton-configuration-h8m6v for pod on switch ip-10-0-0-104.ec2.internal I0424 03:10:12.196384 3689 kube.go:256] Updating pod tsf/tsf-tekton-configuration-h8m6v I0424 03:10:12.207714 3689 pod.go:62] [tsf/tsf-tekton-configuration-h8m6v] pod update took 11.359284ms I0424 03:10:12.207735 3689 base_network_controller_pods.go:951] [default/tsf/tsf-tekton-configuration-h8m6v] addLogicalPort annotation time took 11.385558ms I0424 03:10:12.209013 3689 pods.go:271] [tsf/tsf-tekton-configuration-h8m6v] addLogicalPort took 12.747657ms, libovsdb time 912.945µs I0424 03:10:13.537531 3689 cni.go:452] [tsf/tsf-tekton-configuration-h8m6v 4b57ebe8486f76c49477a1ac09150bc500ba46d7d781befb0a561be24547c91f network default NAD default NAD key ] ADD starting CNI request [tsf/tsf-tekton-configuration-h8m6v 4b57ebe8486f76c49477a1ac09150bc500ba46d7d781befb0a561be24547c91f network default NAD default NAD key ] I0424 03:10:13.546864 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-h8m6v, hostIfaceName: 4b57ebe8486f76c, network: default, NAD default, SandboxID: "4b57ebe8486f76c49477a1ac09150bc500ba46d7d781befb0a561be24547c91f", PCI device ID: , UID: "44722355-1349-4fed-90fa-aeaab277b801", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0424 03:10:13.597905 3689 cni.go:473] [tsf/tsf-tekton-configuration-h8m6v 4b57ebe8486f76c49477a1ac09150bc500ba46d7d781befb0a561be24547c91f network default NAD default NAD key default] ADD finished CNI request [tsf/tsf-tekton-configuration-h8m6v 4b57ebe8486f76c49477a1ac09150bc500ba46d7d781befb0a561be24547c91f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:e9:63:fa:f5:dc\",\"name\":\"4b57ebe8486f76c\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/19e3a5bf-88e7-479e-8656-4a1962839e09\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 03:10:14.217772 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0424 03:10:15.745990 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0424 03:10:19.182018 3689 cni.go:452] [tsf-tas/rekor-createtree-job-88t6m-8kczv a3d14c8012b6902cbcef7d332fd8f0378ca4f5450798dd4253df85dd8d872a48 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-88t6m-8kczv a3d14c8012b6902cbcef7d332fd8f0378ca4f5450798dd4253df85dd8d872a48 network default NAD default NAD key ] I0424 03:10:19.266766 3689 cni.go:473] [tsf-tas/rekor-createtree-job-88t6m-8kczv a3d14c8012b6902cbcef7d332fd8f0378ca4f5450798dd4253df85dd8d872a48 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-88t6m-8kczv a3d14c8012b6902cbcef7d332fd8f0378ca4f5450798dd4253df85dd8d872a48 network default NAD default NAD key default], result "{}", err I0424 03:10:19.316855 3689 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-88t6m-8kczv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:19.316887 3689 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-88t6m-8kczv I0424 03:10:19.318077 3689 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-88t6m-8kczv, ips: 10.129.0.38 I0424 03:10:19.318106 3689 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-88t6m-8kczv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:20.193089 3689 cni.go:452] [tsf/tsf-tekton-configuration-h8m6v 4b57ebe8486f76c49477a1ac09150bc500ba46d7d781befb0a561be24547c91f network default NAD default NAD key ] DEL starting CNI request [tsf/tsf-tekton-configuration-h8m6v 4b57ebe8486f76c49477a1ac09150bc500ba46d7d781befb0a561be24547c91f network default NAD default NAD key ] I0424 03:10:20.205106 3689 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-88t6m-8kczv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:20.205133 3689 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-88t6m-8kczv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:20.268790 3689 cni.go:473] [tsf/tsf-tekton-configuration-h8m6v 4b57ebe8486f76c49477a1ac09150bc500ba46d7d781befb0a561be24547c91f network default NAD default NAD key default] DEL finished CNI request [tsf/tsf-tekton-configuration-h8m6v 4b57ebe8486f76c49477a1ac09150bc500ba46d7d781befb0a561be24547c91f network default NAD default NAD key default], result "{}", err I0424 03:10:20.319024 3689 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-h8m6v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:20.319058 3689 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-h8m6v I0424 03:10:20.320231 3689 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-h8m6v, ips: 10.129.0.40 I0424 03:10:20.320260 3689 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-h8m6v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:21.342810 3689 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-h8m6v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:21.342833 3689 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-h8m6v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:21.382854 3689 obj_retry.go:492] Detected object tsf/patch-tekton-config-rhkbk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:21.382871 3689 obj_retry.go:492] Detected object tsf/patch-tekton-config-rhkbk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:21.391720 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-rhkbk I0424 03:10:21.391746 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-rhkbk I0424 03:10:22.538216 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=435 I0424 03:10:23.257941 3689 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-7bf854c857-zbrbb] creating logical port tsf-tas_rekor-search-ui-7bf854c857-zbrbb for pod on switch ip-10-0-0-104.ec2.internal I0424 03:10:23.258021 3689 kube.go:256] Updating pod tsf-tas/rekor-search-ui-7bf854c857-zbrbb I0424 03:10:23.274300 3689 pod.go:62] [tsf-tas/rekor-search-ui-7bf854c857-zbrbb] pod update took 16.298435ms I0424 03:10:23.274328 3689 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-7bf854c857-zbrbb] addLogicalPort annotation time took 16.329664ms I0424 03:10:23.275440 3689 pods.go:271] [tsf-tas/rekor-search-ui-7bf854c857-zbrbb] addLogicalPort took 17.511594ms, libovsdb time 754.35µs I0424 03:10:23.614829 3689 cni.go:452] [tsf-tas/rekor-search-ui-7bf854c857-zbrbb d0d6e2bce238b120d9b9d343df41aafe8a3160e09c1c2d06c5b16a39c24e0e12 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-7bf854c857-zbrbb d0d6e2bce238b120d9b9d343df41aafe8a3160e09c1c2d06c5b16a39c24e0e12 network default NAD default NAD key ] I0424 03:10:23.624090 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-7bf854c857-zbrbb, hostIfaceName: d0d6e2bce238b12, network: default, NAD default, SandboxID: "d0d6e2bce238b120d9b9d343df41aafe8a3160e09c1c2d06c5b16a39c24e0e12", PCI device ID: , UID: "a114164a-cabe-47d4-b838-96b7ee1b7f77", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0424 03:10:23.673659 3689 cni.go:473] [tsf-tas/rekor-search-ui-7bf854c857-zbrbb d0d6e2bce238b120d9b9d343df41aafe8a3160e09c1c2d06c5b16a39c24e0e12 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-7bf854c857-zbrbb d0d6e2bce238b120d9b9d343df41aafe8a3160e09c1c2d06c5b16a39c24e0e12 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:b0:e1:bf:35:17\",\"name\":\"d0d6e2bce238b12\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c3041881-bc54-4065-bdcd-8d0456f9fefd\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:10:25.423126 3689 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:25.423687 3689 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:25.441911 3689 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:25.441929 3689 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:25.451573 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0424 03:10:25.451597 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0424 03:10:26.141656 3689 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0424 03:10:26.220042 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0424 03:10:26.824868 3689 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-7pd5w] creating logical port tsf-tpa_create-db-7pd5w for pod on switch ip-10-0-0-104.ec2.internal I0424 03:10:26.824953 3689 kube.go:256] Updating pod tsf-tpa/create-db-7pd5w I0424 03:10:26.838677 3689 pod.go:62] [tsf-tpa/create-db-7pd5w] pod update took 13.743182ms I0424 03:10:26.838697 3689 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-7pd5w] addLogicalPort annotation time took 13.769523ms I0424 03:10:26.839924 3689 pods.go:271] [tsf-tpa/create-db-7pd5w] addLogicalPort took 15.070093ms, libovsdb time 833.005µs I0424 03:10:27.183504 3689 cni.go:452] [tsf-tpa/create-db-7pd5w c4a7de909adff4bb298b49cbbc73e95bca3cb6bd1fa955220554eced825c390f network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-7pd5w c4a7de909adff4bb298b49cbbc73e95bca3cb6bd1fa955220554eced825c390f network default NAD default NAD key ] I0424 03:10:27.193933 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-7pd5w, hostIfaceName: c4a7de909adff4b, network: default, NAD default, SandboxID: "c4a7de909adff4bb298b49cbbc73e95bca3cb6bd1fa955220554eced825c390f", PCI device ID: , UID: "c3f436f5-4b65-41b5-ac61-bd99931fdd59", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0424 03:10:27.244468 3689 cni.go:473] [tsf-tpa/create-db-7pd5w c4a7de909adff4bb298b49cbbc73e95bca3cb6bd1fa955220554eced825c390f network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-7pd5w c4a7de909adff4bb298b49cbbc73e95bca3cb6bd1fa955220554eced825c390f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:e2:bb:09:a4:25\",\"name\":\"c4a7de909adff4b\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be6515e1-b536-4ac9-a096-55fe6fd3c4f0\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:10:29.614680 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0424 03:10:31.256590 3689 cni.go:452] [tsf-tpa/create-db-7pd5w c4a7de909adff4bb298b49cbbc73e95bca3cb6bd1fa955220554eced825c390f network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-7pd5w c4a7de909adff4bb298b49cbbc73e95bca3cb6bd1fa955220554eced825c390f network default NAD default NAD key ] I0424 03:10:31.339838 3689 cni.go:473] [tsf-tpa/create-db-7pd5w c4a7de909adff4bb298b49cbbc73e95bca3cb6bd1fa955220554eced825c390f network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-7pd5w c4a7de909adff4bb298b49cbbc73e95bca3cb6bd1fa955220554eced825c390f network default NAD default NAD key default], result "{}", err I0424 03:10:31.391159 3689 obj_retry.go:498] Detected object tsf-tpa/create-db-7pd5w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:31.391197 3689 pods.go:176] Deleting pod: tsf-tpa/create-db-7pd5w I0424 03:10:31.392523 3689 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-7pd5w, ips: 10.129.0.42 I0424 03:10:31.392550 3689 obj_retry.go:498] Detected object tsf-tpa/create-db-7pd5w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:32.415756 3689 obj_retry.go:492] Detected object tsf-tpa/create-db-7pd5w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:32.415775 3689 obj_retry.go:492] Detected object tsf-tpa/create-db-7pd5w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:32.519311 3689 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-nx5td] creating logical port tsf-tpa_migrate-db-nx5td for pod on switch ip-10-0-0-104.ec2.internal I0424 03:10:32.519435 3689 kube.go:256] Updating pod tsf-tpa/migrate-db-nx5td I0424 03:10:32.532875 3689 pod.go:62] [tsf-tpa/migrate-db-nx5td] pod update took 13.462879ms I0424 03:10:32.532898 3689 base_network_controller_pods.go:951] [default/tsf-tpa/migrate-db-nx5td] addLogicalPort annotation time took 13.49158ms I0424 03:10:32.534124 3689 pods.go:271] [tsf-tpa/migrate-db-nx5td] addLogicalPort took 14.829014ms, libovsdb time 858.726µs I0424 03:10:32.878923 3689 cni.go:452] [tsf-tpa/migrate-db-nx5td 4b41fecf7de33685b24267b30e5dc8d0b5da6b4bc4a88ad6f872090c631a22bf network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/migrate-db-nx5td 4b41fecf7de33685b24267b30e5dc8d0b5da6b4bc4a88ad6f872090c631a22bf network default NAD default NAD key ] I0424 03:10:32.889009 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-nx5td, hostIfaceName: 4b41fecf7de3368, network: default, NAD default, SandboxID: "4b41fecf7de33685b24267b30e5dc8d0b5da6b4bc4a88ad6f872090c631a22bf", PCI device ID: , UID: "723e6755-5169-455e-830f-3a516d32f9e3", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0424 03:10:32.938000 3689 cni.go:473] [tsf-tpa/migrate-db-nx5td 4b41fecf7de33685b24267b30e5dc8d0b5da6b4bc4a88ad6f872090c631a22bf network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/migrate-db-nx5td 4b41fecf7de33685b24267b30e5dc8d0b5da6b4bc4a88ad6f872090c631a22bf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:21:b3:97:da:9e\",\"name\":\"4b41fecf7de3368\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79e1708f-0ee6-4ff7-a741-f1abc05e0a28\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 03:10:35.196785 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0424 03:10:35.276870 3689 cni.go:452] [tsf-tpa/migrate-db-nx5td 4b41fecf7de33685b24267b30e5dc8d0b5da6b4bc4a88ad6f872090c631a22bf network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/migrate-db-nx5td 4b41fecf7de33685b24267b30e5dc8d0b5da6b4bc4a88ad6f872090c631a22bf network default NAD default NAD key ] I0424 03:10:35.361003 3689 cni.go:473] [tsf-tpa/migrate-db-nx5td 4b41fecf7de33685b24267b30e5dc8d0b5da6b4bc4a88ad6f872090c631a22bf network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/migrate-db-nx5td 4b41fecf7de33685b24267b30e5dc8d0b5da6b4bc4a88ad6f872090c631a22bf network default NAD default NAD key default], result "{}", err I0424 03:10:35.411216 3689 obj_retry.go:498] Detected object tsf-tpa/migrate-db-nx5td of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:35.411276 3689 pods.go:176] Deleting pod: tsf-tpa/migrate-db-nx5td I0424 03:10:35.412540 3689 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-nx5td, ips: 10.129.0.43 I0424 03:10:35.412566 3689 obj_retry.go:498] Detected object tsf-tpa/migrate-db-nx5td of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:35.555415 3689 obj_retry.go:492] Detected object tsf-tpa/migrate-db-nx5td of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:35.555436 3689 obj_retry.go:492] Detected object tsf-tpa/migrate-db-nx5td of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0424 03:10:38.208653 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0424 03:10:38.697972 3689 base_network_controller_pods.go:487] [default/tsf-tpa/server-5bf6d67cc9-kqkkq] creating logical port tsf-tpa_server-5bf6d67cc9-kqkkq for pod on switch ip-10-0-0-104.ec2.internal I0424 03:10:38.698087 3689 kube.go:256] Updating pod tsf-tpa/server-5bf6d67cc9-kqkkq I0424 03:10:38.713058 3689 pod.go:62] [tsf-tpa/server-5bf6d67cc9-kqkkq] pod update took 15.007049ms I0424 03:10:38.713079 3689 base_network_controller_pods.go:951] [default/tsf-tpa/server-5bf6d67cc9-kqkkq] addLogicalPort annotation time took 15.034867ms I0424 03:10:38.714428 3689 pods.go:271] [tsf-tpa/server-5bf6d67cc9-kqkkq] addLogicalPort took 16.470852ms, libovsdb time 822.642µs I0424 03:10:38.740116 3689 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-dctq6] creating logical port tsf-tpa_importer-86bcdf477b-dctq6 for pod on switch ip-10-0-0-104.ec2.internal I0424 03:10:38.740227 3689 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-dctq6 I0424 03:10:38.753793 3689 pod.go:62] [tsf-tpa/importer-86bcdf477b-dctq6] pod update took 13.588444ms I0424 03:10:38.753816 3689 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-dctq6] addLogicalPort annotation time took 13.615012ms I0424 03:10:38.755083 3689 pods.go:271] [tsf-tpa/importer-86bcdf477b-dctq6] addLogicalPort took 14.990103ms, libovsdb time 881.594µs I0424 03:10:39.517571 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=115 I0424 03:10:39.522100 3689 obj_retry.go:498] Detected object tsf-tpa/create-importers-zjr5n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:39.522692 3689 obj_retry.go:498] Detected object tsf-tpa/create-importers-zjr5n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:10:40.547566 3689 obj_retry.go:492] Detected object tsf-tpa/create-importers-zjr5n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:40.547592 3689 obj_retry.go:492] Detected object tsf-tpa/create-importers-zjr5n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:10:45.970243 3689 cni.go:452] [tsf-tpa/server-5bf6d67cc9-kqkkq 65132503a7eae7c8c0dd24e5316cf313525ebc2592d0d5e2928bfc4a3455cd3c network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-5bf6d67cc9-kqkkq 65132503a7eae7c8c0dd24e5316cf313525ebc2592d0d5e2928bfc4a3455cd3c network default NAD default NAD key ] I0424 03:10:45.980077 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-5bf6d67cc9-kqkkq, hostIfaceName: 65132503a7eae7c, network: default, NAD default, SandboxID: "65132503a7eae7c8c0dd24e5316cf313525ebc2592d0d5e2928bfc4a3455cd3c", PCI device ID: , UID: "3aa26a35-53bc-4163-b6f1-3016592af396", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0424 03:10:46.002255 3689 cni.go:452] [tsf-tpa/importer-86bcdf477b-dctq6 af03f442926b626a6c58f48a84bb23e5eb52e1d7ecb66b824555c38d45ed3f5a network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-dctq6 af03f442926b626a6c58f48a84bb23e5eb52e1d7ecb66b824555c38d45ed3f5a network default NAD default NAD key ] I0424 03:10:46.011610 3689 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-dctq6, hostIfaceName: af03f442926b626, network: default, NAD default, SandboxID: "af03f442926b626a6c58f48a84bb23e5eb52e1d7ecb66b824555c38d45ed3f5a", PCI device ID: , UID: "16aa581b-e53d-4a80-ad43-bc452fd84bf4", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0424 03:10:46.036705 3689 cni.go:473] [tsf-tpa/server-5bf6d67cc9-kqkkq 65132503a7eae7c8c0dd24e5316cf313525ebc2592d0d5e2928bfc4a3455cd3c network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-5bf6d67cc9-kqkkq 65132503a7eae7c8c0dd24e5316cf313525ebc2592d0d5e2928bfc4a3455cd3c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:35:7e:41:7a:f1\",\"name\":\"65132503a7eae7c\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2b763a42-2cc3-452f-9c94-d0f7cc1564ca\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0424 03:10:46.062283 3689 cni.go:473] [tsf-tpa/importer-86bcdf477b-dctq6 af03f442926b626a6c58f48a84bb23e5eb52e1d7ecb66b824555c38d45ed3f5a network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-dctq6 af03f442926b626a6c58f48a84bb23e5eb52e1d7ecb66b824555c38d45ed3f5a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:0b:5d:0c:d6:6d\",\"name\":\"af03f442926b626\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3071cd70-bf2f-4ef0-a5a2-48530126516d\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0424 03:10:46.371399 3689 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0424 03:10:48.741482 3689 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 W0424 03:10:57.378814 3689 util.go:959] Failed to build node endpoints for node ip-10-0-0-104.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0424 03:11:03.668380 3689 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-c4xql-26gcf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:11:03.668867 3689 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-c4xql-26gcf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:11:04.546238 3689 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-c4xql-26gcf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:11:04.546266 3689 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-c4xql-26gcf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:11:14.730428 3689 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:11:14.731185 3689 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:11:14.775747 3689 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:11:14.775770 3689 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:11:14.782560 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0424 03:11:14.782589 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0424 03:11:15.537017 3689 namespace.go:98] [default-tenant] adding namespace I0424 03:11:15.538053 3689 namespace.go:102] [default-tenant] adding namespace took 1.002025ms I0424 03:11:15.538099 3689 namespace.go:98] [build-service] adding namespace I0424 03:11:15.538885 3689 namespace.go:102] [build-service] adding namespace took 769.426µs I0424 03:11:15.549250 3689 namespace.go:98] [integration-service] adding namespace I0424 03:11:15.549264 3689 namespace.go:98] [image-controller] adding namespace I0424 03:11:15.550002 3689 namespace.go:102] [integration-service] adding namespace took 736.329µs I0424 03:11:15.550605 3689 namespace.go:102] [image-controller] adding namespace took 1.329049ms I0424 03:11:15.608135 3689 namespace.go:142] [default-tenant] updating namespace I0424 03:11:15.630275 3689 namespace.go:142] [default-tenant] updating namespace I0424 03:11:15.641249 3689 namespace.go:142] [build-service] updating namespace I0424 03:11:15.674848 3689 namespace.go:142] [build-service] updating namespace I0424 03:11:15.696182 3689 namespace.go:142] [integration-service] updating namespace I0424 03:11:15.725220 3689 namespace.go:142] [integration-service] updating namespace I0424 03:11:15.731229 3689 namespace.go:142] [image-controller] updating namespace I0424 03:11:15.742447 3689 namespace.go:142] [build-service] updating namespace I0424 03:11:15.762494 3689 namespace.go:142] [image-controller] updating namespace I0424 03:11:15.776527 3689 namespace.go:142] [integration-service] updating namespace I0424 03:11:15.792409 3689 namespace.go:98] [release-service] adding namespace I0424 03:11:15.793225 3689 namespace.go:102] [release-service] adding namespace took 800.358µs I0424 03:11:15.833438 3689 namespace.go:142] [release-service] updating namespace I0424 03:11:15.833570 3689 namespace.go:142] [konflux-ui] updating namespace I0424 03:11:15.853293 3689 namespace.go:142] [release-service] updating namespace I0424 03:11:15.926555 3689 namespace.go:98] [konflux-info] adding namespace I0424 03:11:15.927257 3689 namespace.go:102] [konflux-info] adding namespace took 687.281µs I0424 03:11:15.965707 3689 namespace.go:98] [namespace-lister] adding namespace I0424 03:11:15.966448 3689 namespace.go:102] [namespace-lister] adding namespace took 724.349µs I0424 03:11:15.986186 3689 namespace.go:142] [konflux-info] updating namespace I0424 03:11:16.019916 3689 namespace.go:142] [konflux-info] updating namespace I0424 03:11:16.025742 3689 namespace.go:142] [namespace-lister] updating namespace I0424 03:11:16.042536 3689 namespace.go:98] [enterprise-contract-service] adding namespace I0424 03:11:16.043222 3689 namespace.go:102] [enterprise-contract-service] adding namespace took 672.206µs I0424 03:11:16.054936 3689 namespace.go:142] [image-controller] updating namespace I0424 03:11:16.064112 3689 namespace.go:142] [namespace-lister] updating namespace I0424 03:11:16.089648 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0424 03:11:16.149274 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0424 03:11:16.438140 3689 namespace.go:142] [default-tenant] updating namespace I0424 03:11:16.632066 3689 namespace.go:98] [segment-bridge] adding namespace I0424 03:11:16.632819 3689 namespace.go:102] [segment-bridge] adding namespace took 736.337µs I0424 03:11:16.693342 3689 namespace.go:142] [segment-bridge] updating namespace I0424 03:11:16.714218 3689 namespace.go:142] [segment-bridge] updating namespace I0424 03:11:16.784216 3689 namespace.go:142] [build-service] updating namespace I0424 03:11:16.878142 3689 namespace.go:98] [konflux-cli] adding namespace I0424 03:11:16.878959 3689 namespace.go:102] [konflux-cli] adding namespace took 801.971µs I0424 03:11:17.020404 3689 namespace.go:142] [konflux-cli] updating namespace I0424 03:11:17.078721 3689 namespace.go:142] [konflux-cli] updating namespace I0424 03:11:17.222696 3689 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0424 03:11:17.222713 3689 namespace.go:142] [default-tenant] updating namespace I0424 03:11:17.223205 3689 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0424 03:11:17.223221 3689 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0424 03:11:17.225730 3689 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0424 03:11:17.480195 3689 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0424 03:11:17.480217 3689 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0424 03:11:17.481004 3689 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0424 03:11:17.569205 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0424 03:11:17.943285 3689 namespace.go:142] [image-controller] updating namespace I0424 03:11:18.199411 3689 namespace.go:142] [integration-service] updating namespace I0424 03:11:18.628084 3689 namespace.go:142] [konflux-info] updating namespace I0424 03:11:18.813924 3689 namespace.go:142] [namespace-lister] updating namespace I0424 03:11:19.026335 3689 namespace.go:142] [release-service] updating namespace I0424 03:11:19.117367 3689 namespace.go:142] [build-service] updating namespace I0424 03:11:19.143908 3689 namespace.go:142] [default-tenant] updating namespace I0424 03:11:19.166020 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0424 03:11:19.178184 3689 namespace.go:142] [image-controller] updating namespace I0424 03:11:19.189191 3689 namespace.go:142] [integration-service] updating namespace I0424 03:11:19.207466 3689 namespace.go:142] [konflux-info] updating namespace I0424 03:11:19.227140 3689 namespace.go:142] [namespace-lister] updating namespace I0424 03:11:19.239266 3689 namespace.go:142] [release-service] updating namespace I0424 03:11:19.299992 3689 namespace.go:142] [build-service] updating namespace I0424 03:11:19.382142 3689 namespace.go:142] [default-tenant] updating namespace I0424 03:11:19.453304 3689 namespace.go:142] [enterprise-contract-service] updating namespace I0424 03:11:19.533252 3689 namespace.go:142] [image-controller] updating namespace I0424 03:11:19.612717 3689 namespace.go:142] [integration-service] updating namespace I0424 03:11:19.736805 3689 namespace.go:142] [konflux-info] updating namespace I0424 03:11:19.833108 3689 namespace.go:142] [namespace-lister] updating namespace I0424 03:11:19.914725 3689 namespace.go:142] [release-service] updating namespace W0424 03:11:20.691044 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0424 03:11:21.056969 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0424 03:11:21.177957 3689 namespace.go:142] [konflux-cli] updating namespace I0424 03:11:21.375977 3689 namespace.go:142] [segment-bridge] updating namespace I0424 03:11:21.474945 3689 namespace.go:142] [konflux-cli] updating namespace I0424 03:11:21.503618 3689 namespace.go:142] [segment-bridge] updating namespace I0424 03:11:21.581724 3689 namespace.go:142] [konflux-cli] updating namespace I0424 03:11:21.699085 3689 namespace.go:142] [segment-bridge] updating namespace W0424 03:11:26.694362 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0424 03:11:26.756174 3689 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0424 03:11:28.486308 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0424 03:11:28.486336 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0424 03:11:29.499503 3689 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0424 03:11:29.616175 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 W0424 03:11:29.718228 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0424 03:11:29.718283 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0424 03:11:33.751324 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0424 03:11:33.751344 3689 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0424 03:11:34.545087 3689 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0424 03:11:34.545104 3689 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0424 03:11:34.580376 3689 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0424 03:11:34.580588 3689 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service I0424 03:11:41.169700 3689 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-bbv2x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:11:41.170281 3689 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-bbv2x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:11:41.795648 3689 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-bbv2x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:11:41.795672 3689 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-bbv2x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:11:41.804833 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-bbv2x I0424 03:11:41.804854 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-bbv2x I0424 03:11:56.745965 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0424 03:12:39.519980 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=76 I0424 03:12:42.718989 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0424 03:12:44.446365 3689 obj_retry.go:498] Detected object konflux-ui/proxy-6d4ccf457d-kcnrh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:12:44.447034 3689 obj_retry.go:498] Detected object konflux-ui/proxy-6d4ccf457d-kcnrh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:12:44.906568 3689 obj_retry.go:492] Detected object konflux-ui/proxy-6d4ccf457d-kcnrh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:12:44.906591 3689 obj_retry.go:492] Detected object konflux-ui/proxy-6d4ccf457d-kcnrh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:12:44.916366 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-6d4ccf457d-kcnrh I0424 03:12:44.916386 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-6d4ccf457d-kcnrh I0424 03:12:48.336909 3689 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:12:48.337455 3689 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:12:48.357115 3689 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:12:48.357132 3689 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:12:48.363950 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0424 03:12:48.363973 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0424 03:13:19.093625 3689 namespace.go:98] [default-managed-tenant-m2emy] adding namespace I0424 03:13:19.094493 3689 namespace.go:102] [default-managed-tenant-m2emy] adding namespace took 848.548µs I0424 03:13:19.121670 3689 namespace.go:142] [default-managed-tenant-m2emy] updating namespace I0424 03:13:19.140245 3689 namespace.go:142] [default-managed-tenant-m2emy] updating namespace I0424 03:13:20.142543 3689 namespace.go:142] [default-managed-tenant-m2emy] updating namespace I0424 03:13:20.274234 3689 namespace.go:142] [default-managed-tenant-m2emy] updating namespace I0424 03:13:20.375246 3689 namespace.go:142] [default-managed-tenant-m2emy] updating namespace I0424 03:13:26.523545 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=170 I0424 03:14:19.730759 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0424 03:14:48.622876 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0424 03:14:59.519858 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=12 I0424 03:15:29.725896 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0424 03:15:38.736778 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0424 03:15:50.744451 3689 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0424 03:16:41.540606 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=196 I0424 03:17:08.617653 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0424 03:17:21.748628 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0424 03:17:51.523950 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=49 I0424 03:18:44.618945 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0424 03:19:20.520563 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=54 I0424 03:19:41.748672 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0424 03:20:03.830665 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616680-855mr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0424 03:20:03.831164 3689 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616680-855mr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0424 03:20:04.702709 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616680-855mr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:20:04.702734 3689 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29616680-855mr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:20:25.721422 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0424 03:20:30.525782 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=9 I0424 03:20:32.063214 3689 obj_retry.go:492] Detected object tsf-tpa/create-db-7pd5w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:20:32.063263 3689 obj_retry.go:492] Detected object tsf-tpa/create-db-7pd5w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:20:32.069304 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-7pd5w I0424 03:20:32.069332 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-7pd5w I0424 03:20:35.041655 3689 obj_retry.go:492] Detected object tsf-tpa/migrate-db-nx5td of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:20:35.041702 3689 obj_retry.go:492] Detected object tsf-tpa/migrate-db-nx5td of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:20:35.046870 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-nx5td I0424 03:20:35.046899 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-nx5td I0424 03:20:40.032028 3689 obj_retry.go:492] Detected object tsf-tpa/create-importers-zjr5n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:20:40.032077 3689 obj_retry.go:492] Detected object tsf-tpa/create-importers-zjr5n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0424 03:20:40.037999 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-zjr5n I0424 03:20:40.038024 3689 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-zjr5n I0424 03:21:33.733203 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0424 03:21:55.521931 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=39 I0424 03:22:01.522923 3689 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0424 03:22:14.625859 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0424 03:22:42.739221 3689 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0424 03:23:46.746890 3689 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0424 03:24:38.620152 3689 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9