+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-130.ec2.internal ++ [[ -n ip-10-0-1-130.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-130.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0425 19:00:14.571826923 - disable conntrack on geneve port' I0425 19:00:14.571826923 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0425 19:00:14.590012658 - starting ovnkube-node + echo 'I0425 19:00:14.590012658 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-130.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-130.ec2.internal --init-node ip-10-0-1-130.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-130.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0425 19:00:14.630450 3656 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0425 19:00:14.630514 3656 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-5d4a611332.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0425 19:00:14.632440 3656 kube.go:431] Waiting for certificate I0425 19:00:14.632466 3656 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0425 19:00:14.632518 3656 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0425 19:00:14.633282 3656 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0425 19:00:14.688938 3656 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0425 19:00:14.688961 3656 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0425 19:00:14.694073 3656 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0425 19:00:14.694169 3656 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-5k2hw" I0425 19:00:14.706028 3656 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-5k2hw" I0425 19:00:14.706138 3656 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0425 19:00:15.633112 3656 kube.go:438] Certificate found I0425 19:00:15.633684 3656 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0425 19:00:15.634707 3656 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0425 19:00:15.702498 3656 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0425 19:00:15.709285 3656 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-26 18:55:14 +0000 UTC" deadline="2026-04-26 13:38:20.544794495 +0000 UTC" I0425 19:00:15.709380 3656 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="18h38m4.83542553s" I0425 19:00:15.717494 3656 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0425 19:00:15.717933 3656 server.go:101] MetricServer registers OVS metrics I0425 19:00:15.718789 3656 server.go:105] MetricServer registers OVN DB metrics I0425 19:00:15.720560 3656 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0425 19:00:15.720616 3656 factory.go:561] Starting watch factory I0425 19:00:15.720682 3656 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.720687 3656 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.720693 3656 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.720698 3656 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.720707 3656 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.720721 3656 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.720894 3656 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.720937 3656 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.720967 3656 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.720989 3656 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.721409 3656 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.721430 3656 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.723578 3656 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0425 19:00:15.723600 3656 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0425 19:00:15.723703 3656 controller_manager.go:370] Starting the ovnkube controller I0425 19:00:15.723716 3656 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-130.ec2.internal I0425 19:00:15.723762 3656 controller_manager.go:395] NBDB zone sync took: 38.916µs I0425 19:00:15.723773 3656 factory.go:561] Starting watch factory I0425 19:00:15.736771 3656 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0425 19:00:15.736791 3656 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0425 19:00:15.739081 3656 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.740682 3656 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0425 19:00:15.740713 3656 server.go:109] MetricServer registers OVN Controller metrics I0425 19:00:15.741425 3656 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.741623 3656 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.743356 3656 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.746388 3656 server.go:113] MetricServer registers OVN Northd metrics I0425 19:00:15.747871 3656 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.750764 3656 metrics.go:519] OVN Metrics Server starts to run ... I0425 19:00:15.752414 3656 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0425 19:00:15.826214 3656 factory.go:1977] *v1.Service informer cache synced successfully I0425 19:00:15.826240 3656 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0425 19:00:15.826245 3656 factory.go:1977] *v1.Pod informer cache synced successfully I0425 19:00:15.826248 3656 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0425 19:00:15.826252 3656 factory.go:1977] *v1.Namespace informer cache synced successfully I0425 19:00:15.826255 3656 factory.go:1977] *v1.Node informer cache synced successfully I0425 19:00:15.826345 3656 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 19:00:15.826366 3656 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 19:00:15.826345 3656 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 19:00:15.826395 3656 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 19:00:15.829820 3656 factory.go:1977] *v1.Service informer cache synced successfully I0425 19:00:15.829834 3656 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0425 19:00:15.829838 3656 factory.go:1977] *v1.Pod informer cache synced successfully I0425 19:00:15.829841 3656 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0425 19:00:15.829846 3656 factory.go:1977] *v1.Namespace informer cache synced successfully I0425 19:00:15.829849 3656 factory.go:1977] *v1.Node informer cache synced successfully I0425 19:00:15.830020 3656 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 19:00:15.830019 3656 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0425 19:00:15.831988 3656 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0425 19:00:15.831995 3656 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0425 19:00:15.832066 3656 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.832079 3656 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.835207 3656 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0425 19:00:15.835219 3656 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0425 19:00:15.835317 3656 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.837332 3656 factory.go:1977] *v1.EgressIP informer cache synced successfully I0425 19:00:15.837405 3656 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.837438 3656 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.840305 3656 factory.go:1977] *v1.EgressIP informer cache synced successfully I0425 19:00:15.840694 3656 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.842616 3656 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0425 19:00:15.842692 3656 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.842702 3656 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.845550 3656 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0425 19:00:15.845967 3656 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.847974 3656 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0425 19:00:15.848029 3656 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.848052 3656 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.850766 3656 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0425 19:00:15.851319 3656 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.853318 3656 factory.go:1977] *v1.EgressService informer cache synced successfully I0425 19:00:15.853373 3656 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.853385 3656 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.855986 3656 factory.go:1977] *v1.EgressService informer cache synced successfully I0425 19:00:15.856350 3656 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.858850 3656 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0425 19:00:15.859062 3656 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0425 19:00:15.859080 3656 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0425 19:00:15.861885 3656 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0425 19:00:15.862261 3656 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0425 19:00:15.864240 3656 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0425 19:00:15.864317 3656 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.864334 3656 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.864380 3656 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.864422 3656 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.867101 3656 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0425 19:00:15.867342 3656 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.867607 3656 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0425 19:00:15.869606 3656 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0425 19:00:15.869620 3656 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0425 19:00:15.869627 3656 factory.go:680] Watch Factory start up complete, took: 149.013055ms I0425 19:00:15.869705 3656 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0425 19:00:15.872134 3656 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0425 19:00:15.872158 3656 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0425 19:00:15.872170 3656 factory.go:680] Watch Factory start up complete, took: 148.394242ms I0425 19:00:15.872180 3656 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-1-130.ec2.internal" zone I0425 19:00:15.916011 3656 default_node_network_controller.go:710] Initializing the default node network controller I0425 19:00:16.002416 3656 udn_isolation.go:101] Starting UDN host isolation manager I0425 19:00:16.008397 3656 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0425 19:00:16.068389 3656 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0425 19:00:16.068487 3656 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0425 19:00:16.068507 3656 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0425 19:00:16.101861 3656 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0425 19:00:16.101928 3656 default_node_network_controller.go:809] Node ip-10-0-1-130.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0425 19:00:16.107076 3656 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.130"] k8s.ovn.org/zone-name:ip-10-0-1-130.ec2.internal] on node ip-10-0-1-130.ec2.internal I0425 19:00:16.130620 3656 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0425 19:00:16.137851 3656 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0425 19:00:16.137998 3656 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0425 19:00:16.138161 3656 gateway_init.go:263] Preparing Gateway I0425 19:00:16.138173 3656 gateway_shared_intf.go:1687] Creating new gateway I0425 19:00:16.255740 3656 iptables.go:108] Creating table: filter chain: FORWARD I0425 19:00:16.261756 3656 iptables.go:108] Creating table: filter chain: OUTPUT I0425 19:00:16.267762 3656 gateway_shared_intf.go:1816] Gateway Creation Complete I0425 19:00:16.267782 3656 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-130.ec2.internal","mac-address":"0a:ff:cc:8f:5b:35","ip-addresses":["10.0.1.130/24"],"ip-address":"10.0.1.130/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:f0ce6ac8-41d6-4462-9c7a-946ed0deee40 k8s.ovn.org/node-encap-ips:["10.0.1.130"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.130/24"} k8s.ovn.org/zone-name:ip-10-0-1-130.ec2.internal] on node ip-10-0-1-130.ec2.internal I0425 19:00:16.289717 3656 default_node_network_controller.go:519] Node connection status = connected I0425 19:00:16.372426 3656 controller_manager.go:429] Waiting for node in zone sync took: 500.229383ms I0425 19:00:16.382982 3656 controller_manager.go:611] Switching to transit router for layer2 networks I0425 19:00:16.383003 3656 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-130.ec2.internal I0425 19:00:16.400011 3656 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0425 19:00:16.407434 3656 controller_manager.go:290] SCTP support detected in OVN I0425 19:00:16.407997 3656 services_controller.go:69] Creating services controller for network=default I0425 19:00:16.408139 3656 udn_enabled_svc.go:80] Waiting for service informer to sync I0425 19:00:16.408263 3656 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0425 19:00:16.408357 3656 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0425 19:00:16.408377 3656 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0425 19:00:16.408390 3656 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0425 19:00:16.408574 3656 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0425 19:00:16.408644 3656 udn_enabled_svc.go:95] Performing full resync I0425 19:00:16.408670 3656 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 58.219µs I0425 19:00:16.408673 3656 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0425 19:00:16.408719 3656 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 29.662µs I0425 19:00:16.408728 3656 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 54.954µs I0425 19:00:16.408733 3656 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0425 19:00:16.408741 3656 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0425 19:00:16.408749 3656 default_network_controller.go:325] Starting the default network controller I0425 19:00:16.408764 3656 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 13.764µs I0425 19:00:16.408919 3656 udn_enabled_svc.go:99] Waiting for handler to sync I0425 19:00:16.408930 3656 udn_enabled_svc.go:104] Starting worker I0425 19:00:16.417965 3656 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0425 19:00:16.418022 3656 repair.go:33] Syncing exgw routes took 44.774µs I0425 19:00:16.418065 3656 default_network_controller.go:414] Starting all the Watchers... I0425 19:00:16.418219 3656 namespace.go:98] [openshift-ingress] adding namespace I0425 19:00:16.418226 3656 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0425 19:00:16.418222 3656 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0425 19:00:16.418225 3656 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0425 19:00:16.418235 3656 namespace.go:98] [openshift-network-console] adding namespace I0425 19:00:16.418240 3656 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0425 19:00:16.418239 3656 namespace.go:98] [openshift-backplane-csm] adding namespace I0425 19:00:16.418245 3656 namespace.go:98] [dedicated-admin] adding namespace I0425 19:00:16.418264 3656 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0425 19:00:16.418272 3656 namespace.go:98] [openshift-operators] adding namespace I0425 19:00:16.418275 3656 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0425 19:00:16.418279 3656 namespace.go:98] [kube-system] adding namespace I0425 19:00:16.418280 3656 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0425 19:00:16.418285 3656 namespace.go:98] [openshift-backplane-cee] adding namespace I0425 19:00:16.418318 3656 namespace.go:98] [openshift-console] adding namespace I0425 19:00:16.418798 3656 namespace.go:102] [openshift-ingress] adding namespace took 565.412µs I0425 19:00:16.418813 3656 namespace.go:98] [openshift-host-network] adding namespace I0425 19:00:16.419376 3656 namespace.go:102] [openshift-host-network] adding namespace took 557.878µs I0425 19:00:16.419390 3656 namespace.go:98] [openshift-controller-manager] adding namespace I0425 19:00:16.419929 3656 namespace.go:102] [openshift-controller-manager] adding namespace took 531.708µs I0425 19:00:16.419944 3656 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0425 19:00:16.420480 3656 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 2.245775ms I0425 19:00:16.420494 3656 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0425 19:00:16.420961 3656 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 2.722461ms I0425 19:00:16.420975 3656 namespace.go:98] [openshift-customer-monitoring] adding namespace I0425 19:00:16.421481 3656 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 3.238775ms I0425 19:00:16.421493 3656 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0425 19:00:16.421920 3656 namespace.go:102] [openshift-network-console] adding namespace took 3.677334ms I0425 19:00:16.421930 3656 namespace.go:98] [openshift-backplane-mobb] adding namespace I0425 19:00:16.422399 3656 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 4.151801ms I0425 19:00:16.422412 3656 namespace.go:98] [openshift-kube-apiserver] adding namespace I0425 19:00:16.422824 3656 namespace.go:102] [openshift-backplane-csm] adding namespace took 4.575785ms I0425 19:00:16.422833 3656 namespace.go:98] [openshift-ingress-operator] adding namespace I0425 19:00:16.423661 3656 namespace.go:102] [dedicated-admin] adding namespace took 5.405924ms I0425 19:00:16.423674 3656 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0425 19:00:16.424130 3656 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.856666ms I0425 19:00:16.424145 3656 namespace.go:98] [openshift-marketplace] adding namespace I0425 19:00:16.424656 3656 namespace.go:102] [openshift-operators] adding namespace took 6.375811ms I0425 19:00:16.424669 3656 namespace.go:98] [openshift-route-controller-manager] adding namespace I0425 19:00:16.425188 3656 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.903003ms I0425 19:00:16.425206 3656 namespace.go:98] [openshift-dns-operator] adding namespace I0425 19:00:16.425655 3656 namespace.go:102] [kube-system] adding namespace took 7.367052ms I0425 19:00:16.425672 3656 namespace.go:98] [openshift-backplane-tam] adding namespace I0425 19:00:16.426154 3656 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.863624ms I0425 19:00:16.426168 3656 namespace.go:98] [openshift-ingress-canary] adding namespace I0425 19:00:16.426712 3656 namespace.go:102] [openshift-backplane-cee] adding namespace took 8.419009ms I0425 19:00:16.426726 3656 namespace.go:98] [openshift-multus] adding namespace I0425 19:00:16.427185 3656 namespace.go:102] [openshift-console] adding namespace took 8.858291ms I0425 19:00:16.427198 3656 namespace.go:98] [openshift-machine-config-operator] adding namespace I0425 19:00:16.427618 3656 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.667148ms I0425 19:00:16.427634 3656 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0425 19:00:16.428098 3656 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.599872ms I0425 19:00:16.428109 3656 namespace.go:98] [openshift-monitoring] adding namespace I0425 19:00:16.428608 3656 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.628468ms I0425 19:00:16.428619 3656 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0425 19:00:16.429019 3656 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.521144ms I0425 19:00:16.429034 3656 namespace.go:98] [openshift-package-operator] adding namespace I0425 19:00:16.429430 3656 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.495367ms I0425 19:00:16.429441 3656 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0425 19:00:16.429846 3656 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.428556ms I0425 19:00:16.429858 3656 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0425 19:00:16.430293 3656 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.454903ms I0425 19:00:16.430304 3656 namespace.go:98] [openshift-service-ca-operator] adding namespace I0425 19:00:16.430691 3656 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.011313ms I0425 19:00:16.430703 3656 namespace.go:98] [openshift-network-node-identity] adding namespace I0425 19:00:16.431170 3656 namespace.go:102] [openshift-marketplace] adding namespace took 7.020392ms I0425 19:00:16.431180 3656 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0425 19:00:16.431721 3656 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.046358ms I0425 19:00:16.431741 3656 namespace.go:98] [openshift-backplane] adding namespace I0425 19:00:16.432208 3656 namespace.go:102] [openshift-dns-operator] adding namespace took 6.996966ms I0425 19:00:16.432221 3656 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0425 19:00:16.432704 3656 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.023087ms I0425 19:00:16.432723 3656 namespace.go:98] [openshift-backplane-srep-ro] adding namespace W0425 19:00:16.433142 3656 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-jszsp: pod openshift-multus/network-metrics-daemon-jszsp: no pod IPs found W0425 19:00:16.433155 3656 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-r9tw4: pod openshift-multus/network-metrics-daemon-r9tw4: no pod IPs found W0425 19:00:16.433160 3656 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-rj8j4: pod openshift-multus/network-metrics-daemon-rj8j4: no pod IPs found I0425 19:00:16.433182 3656 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.008495ms I0425 19:00:16.433195 3656 namespace.go:98] [openshift-backplane-cse] adding namespace I0425 19:00:16.433603 3656 namespace.go:102] [openshift-multus] adding namespace took 6.872194ms I0425 19:00:16.433615 3656 namespace.go:98] [openshift-network-operator] adding namespace I0425 19:00:16.434024 3656 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.821474ms I0425 19:00:16.434053 3656 namespace.go:98] [openshift-operators-redhat] adding namespace I0425 19:00:16.434473 3656 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.833325ms I0425 19:00:16.435083 3656 namespace.go:102] [openshift-monitoring] adding namespace took 6.967973ms I0425 19:00:16.435097 3656 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0425 19:00:16.435521 3656 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.897264ms I0425 19:00:16.435532 3656 namespace.go:98] [openshift-machine-api] adding namespace I0425 19:00:16.435902 3656 namespace.go:102] [openshift-package-operator] adding namespace took 6.849406ms I0425 19:00:16.435917 3656 namespace.go:98] [open-cluster-management-2ptbrbue1ohsm606gsl9i41bgt475dp0] adding namespace I0425 19:00:16.436297 3656 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.851636ms I0425 19:00:16.436310 3656 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0425 19:00:16.436657 3656 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.794398ms I0425 19:00:16.436668 3656 namespace.go:98] [openshift-config-managed] adding namespace I0425 19:00:16.437002 3656 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.69267ms I0425 19:00:16.437016 3656 namespace.go:98] [openshift-console-operator] adding namespace I0425 19:00:16.437438 3656 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.729784ms I0425 19:00:16.437450 3656 namespace.go:98] [openshift-insights] adding namespace I0425 19:00:16.437776 3656 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.59103ms I0425 19:00:16.437787 3656 namespace.go:98] [openshift-must-gather-operator] adding namespace I0425 19:00:16.438241 3656 namespace.go:102] [openshift-backplane] adding namespace took 6.491038ms I0425 19:00:16.438255 3656 namespace.go:98] [openshift-node] adding namespace I0425 19:00:16.438687 3656 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.46108ms I0425 19:00:16.438699 3656 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0425 19:00:16.439204 3656 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.475374ms I0425 19:00:16.439216 3656 namespace.go:98] [kube-public] adding namespace I0425 19:00:16.439590 3656 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.389548ms I0425 19:00:16.439603 3656 namespace.go:98] [openshift-authentication] adding namespace I0425 19:00:16.439974 3656 namespace.go:102] [openshift-network-operator] adding namespace took 6.354772ms I0425 19:00:16.439985 3656 namespace.go:98] [openshift-backplane-srep] adding namespace I0425 19:00:16.440430 3656 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.372074ms I0425 19:00:16.440439 3656 namespace.go:98] [openshift] adding namespace I0425 19:00:16.440857 3656 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.754535ms I0425 19:00:16.440870 3656 namespace.go:98] [openshift-kube-scheduler] adding namespace I0425 19:00:16.441291 3656 namespace.go:102] [openshift-machine-api] adding namespace took 5.754151ms I0425 19:00:16.441305 3656 namespace.go:98] [openshift-infra] adding namespace I0425 19:00:16.441693 3656 namespace.go:102] [open-cluster-management-2ptbrbue1ohsm606gsl9i41bgt475dp0] adding namespace took 5.771204ms I0425 19:00:16.441705 3656 namespace.go:98] [openshift-authentication-operator] adding namespace I0425 19:00:16.442077 3656 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 5.761882ms I0425 19:00:16.442089 3656 namespace.go:98] [openshift-console-user-settings] adding namespace I0425 19:00:16.442527 3656 namespace.go:102] [openshift-config-managed] adding namespace took 5.853069ms I0425 19:00:16.442541 3656 namespace.go:98] [openshift-cluster-version] adding namespace I0425 19:00:16.443254 3656 namespace.go:102] [openshift-console-operator] adding namespace took 6.233161ms I0425 19:00:16.443268 3656 namespace.go:98] [openshift-network-diagnostics] adding namespace I0425 19:00:16.443683 3656 namespace.go:102] [openshift-insights] adding namespace took 6.229051ms I0425 19:00:16.443692 3656 namespace.go:98] [openshift-config] adding namespace I0425 19:00:16.444112 3656 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.321474ms I0425 19:00:16.444121 3656 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0425 19:00:16.444575 3656 namespace.go:102] [openshift-node] adding namespace took 6.314661ms I0425 19:00:16.444585 3656 namespace.go:98] [openshift-apiserver] adding namespace I0425 19:00:16.444989 3656 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.284669ms I0425 19:00:16.445001 3656 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0425 19:00:16.445431 3656 namespace.go:102] [kube-public] adding namespace took 6.20831ms I0425 19:00:16.445445 3656 namespace.go:98] [openshift-etcd] adding namespace I0425 19:00:16.445827 3656 namespace.go:102] [openshift-authentication] adding namespace took 6.219085ms I0425 19:00:16.445838 3656 namespace.go:98] [openshift-config-operator] adding namespace I0425 19:00:16.446241 3656 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.251388ms I0425 19:00:16.446253 3656 namespace.go:98] [openshift-image-registry] adding namespace I0425 19:00:16.446660 3656 namespace.go:102] [openshift] adding namespace took 6.216195ms I0425 19:00:16.446673 3656 namespace.go:98] [openshift-logging] adding namespace I0425 19:00:16.447168 3656 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.290369ms I0425 19:00:16.447186 3656 namespace.go:98] [openshift-apiserver-operator] adding namespace I0425 19:00:16.447694 3656 namespace.go:102] [openshift-infra] adding namespace took 6.383496ms I0425 19:00:16.448114 3656 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.403553ms I0425 19:00:16.448556 3656 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.461635ms W0425 19:00:16.448933 3656 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-2qtwp: pod openshift-network-diagnostics/network-check-target-2qtwp: no pod IPs found W0425 19:00:16.448945 3656 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-gwq25: pod openshift-network-diagnostics/network-check-target-gwq25: no pod IPs found W0425 19:00:16.448952 3656 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-hblkv: pod openshift-network-diagnostics/network-check-target-hblkv: no pod IPs found I0425 19:00:16.448973 3656 namespace.go:102] [openshift-cluster-version] adding namespace took 6.429183ms I0425 19:00:16.449431 3656 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.157916ms I0425 19:00:16.449807 3656 namespace.go:102] [openshift-config] adding namespace took 6.109545ms I0425 19:00:16.450228 3656 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.102886ms I0425 19:00:16.450608 3656 namespace.go:102] [openshift-apiserver] adding namespace took 6.018185ms I0425 19:00:16.451011 3656 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.005515ms I0425 19:00:16.451428 3656 namespace.go:102] [openshift-etcd] adding namespace took 5.977259ms I0425 19:00:16.451441 3656 namespace.go:98] [openshift-dns] adding namespace I0425 19:00:16.451824 3656 namespace.go:102] [openshift-config-operator] adding namespace took 5.97981ms I0425 19:00:16.452231 3656 namespace.go:102] [openshift-image-registry] adding namespace took 5.9728ms I0425 19:00:16.452243 3656 namespace.go:98] [kube-node-lease] adding namespace I0425 19:00:16.452615 3656 namespace.go:102] [openshift-logging] adding namespace took 5.937524ms I0425 19:00:16.452627 3656 namespace.go:98] [default] adding namespace I0425 19:00:16.452993 3656 namespace.go:102] [openshift-apiserver-operator] adding namespace took 5.802325ms I0425 19:00:16.453395 3656 namespace.go:102] [openshift-dns] adding namespace took 1.94998ms I0425 19:00:16.453789 3656 namespace.go:102] [kube-node-lease] adding namespace took 1.541527ms I0425 19:00:16.454138 3656 namespace.go:102] [default] adding namespace took 1.507057ms W0425 19:00:16.454217 3656 master.go:471] Unable to parse remote node ip-10-0-0-74.ec2.internal chassis-id annotation. Chassis may be removed during sync W0425 19:00:16.454226 3656 master.go:471] Unable to parse remote node ip-10-0-2-191.ec2.internal chassis-id annotation. Chassis may be removed during sync I0425 19:00:16.454320 3656 master.go:548] Adding or Updating local node "ip-10-0-1-130.ec2.internal" for network "default" I0425 19:00:16.454370 3656 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-191.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-191.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-191.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-191.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-191.ec2.internal I0425 19:00:16.454371 3656 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-74.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-74.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-74.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-74.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-74.ec2.internal W0425 19:00:16.461023 3656 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-jszsp: pod openshift-multus/network-metrics-daemon-jszsp: no pod IPs found W0425 19:00:16.461062 3656 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-hblkv: pod openshift-network-diagnostics/network-check-target-hblkv: no pod IPs found I0425 19:00:16.462097 3656 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-1-130.ec2.internal for the network default I0425 19:00:16.462360 3656 zone_ic_handler.go:195] Time taken to create transit switch: 253.51µs I0425 19:00:16.463003 3656 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0425 19:00:16.463033 3656 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0425 19:00:16.463068 3656 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0425 19:00:16.463068 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-74.ec2.internal in network "default" I0425 19:00:16.463116 3656 node_tracker.go:235] Node ip-10-0-0-74.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-74.ec2.internal" I0425 19:00:16.463128 3656 node_tracker.go:169] Node ip-10-0-0-74.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:16.463141 3656 services_controller.go:574] Full service sync requested for network=default I0425 19:00:16.463150 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-130.ec2.internal in network "default" I0425 19:00:16.463178 3656 node_tracker.go:169] Node ip-10-0-1-130.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:16.463186 3656 services_controller.go:574] Full service sync requested for network=default I0425 19:00:16.463193 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-191.ec2.internal in network "default" I0425 19:00:16.463202 3656 node_tracker.go:235] Node ip-10-0-2-191.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-191.ec2.internal" I0425 19:00:16.463208 3656 node_tracker.go:169] Node ip-10-0-2-191.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:16.463211 3656 services_controller.go:574] Full service sync requested for network=default I0425 19:00:16.563696 3656 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0425 19:00:16.563719 3656 services_controller.go:208] Setting up event handlers for services for network=default I0425 19:00:16.563805 3656 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0425 19:00:16.563869 3656 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0425 19:00:16.563895 3656 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0425 19:00:16.664380 3656 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0425 19:00:16.664399 3656 repair.go:60] Starting repairing loop for services I0425 19:00:16.664477 3656 repair.go:131] Deleted 0 stale service LBs I0425 19:00:16.664495 3656 repair.go:137] Deleted 0 stale Chassis Template Vars I0425 19:00:16.664510 3656 repair.go:62] Finished repairing loop for services: 112.688µs I0425 19:00:16.664554 3656 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0425 19:00:16.664565 3656 services_controller.go:254] Starting workers for network=default I0425 19:00:16.665130 3656 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-jszsp] creating logical port openshift-multus_network-metrics-daemon-jszsp for pod on switch ip-10-0-1-130.ec2.internal I0425 19:00:16.665154 3656 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-gwq25, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-gwq25: suppressed error logged: pod openshift-network-diagnostics/network-check-target-gwq25: no pod IPs found I0425 19:00:16.665220 3656 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-r9tw4, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-r9tw4: suppressed error logged: pod openshift-multus/network-metrics-daemon-r9tw4: no pod IPs found I0425 19:00:16.665274 3656 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-rj8j4, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-rj8j4: suppressed error logged: pod openshift-multus/network-metrics-daemon-rj8j4: no pod IPs found I0425 19:00:16.665312 3656 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-jszsp I0425 19:00:16.665330 3656 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-hblkv] creating logical port openshift-network-diagnostics_network-check-target-hblkv for pod on switch ip-10-0-1-130.ec2.internal I0425 19:00:16.665377 3656 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-2qtwp, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-2qtwp: suppressed error logged: pod openshift-network-diagnostics/network-check-target-2qtwp: no pod IPs found I0425 19:00:16.665642 3656 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-hblkv I0425 19:00:16.683670 3656 pod.go:62] [openshift-multus/network-metrics-daemon-jszsp] pod update took 18.444753ms I0425 19:00:16.683692 3656 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-jszsp] addLogicalPort annotation time took 18.470598ms I0425 19:00:16.684790 3656 pods.go:271] [openshift-multus/network-metrics-daemon-jszsp] addLogicalPort took 19.677378ms, libovsdb time 843.957µs I0425 19:00:16.688527 3656 pod.go:62] [openshift-network-diagnostics/network-check-target-hblkv] pod update took 22.958695ms I0425 19:00:16.688601 3656 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-hblkv] addLogicalPort annotation time took 23.031552ms I0425 19:00:16.689689 3656 pods.go:271] [openshift-network-diagnostics/network-check-target-hblkv] addLogicalPort took 24.36478ms, libovsdb time 851.753µs I0425 19:00:16.690251 3656 admin_network_policy_controller.go:227] Starting controller default-network-controller I0425 19:00:16.690280 3656 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0425 19:00:16.690307 3656 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0425 19:00:16.690313 3656 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0425 19:00:16.690558 3656 repair.go:29] Repairing admin network policies took 237.701µs I0425 19:00:16.690779 3656 repair.go:92] Repairing baseline admin network policies took 166.133µs I0425 19:00:16.691217 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0425 19:00:16.691244 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0425 19:00:16.694279 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0425 19:00:16.694312 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0425 19:00:16.694323 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0425 19:00:16.695812 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0425 19:00:16.695831 3656 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0425 19:00:16.695840 3656 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0425 19:00:16.697486 3656 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0425 19:00:16.697506 3656 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0425 19:00:16.697516 3656 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0425 19:00:16.698825 3656 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0425 19:00:16.698844 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0425 19:00:16.698854 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0425 19:00:16.699867 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0425 19:00:16.699886 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0425 19:00:16.699896 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0425 19:00:16.701609 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0425 19:00:16.701628 3656 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0425 19:00:16.701637 3656 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0425 19:00:16.702676 3656 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0425 19:00:16.702693 3656 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0425 19:00:16.702700 3656 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0425 19:00:16.703189 3656 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0425 19:00:16.703207 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0425 19:00:16.703217 3656 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0425 19:00:16.705219 3656 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0425 19:00:16.705236 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0425 19:00:16.705245 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0425 19:00:16.705718 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0425 19:00:16.705731 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0425 19:00:16.705739 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0425 19:00:16.706238 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0425 19:00:16.706257 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0425 19:00:16.706265 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0425 19:00:16.706719 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0425 19:00:16.706735 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0425 19:00:16.706741 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0425 19:00:16.708005 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0425 19:00:16.708021 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0425 19:00:16.708028 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0425 19:00:16.708322 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0425 19:00:16.708340 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0425 19:00:16.708728 3656 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0425 19:00:16.708745 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0425 19:00:16.709457 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0425 19:00:16.709476 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0425 19:00:16.709484 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0425 19:00:16.709978 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0425 19:00:16.709995 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0425 19:00:16.710003 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0425 19:00:16.710568 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0425 19:00:16.710586 3656 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0425 19:00:16.710594 3656 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0425 19:00:16.711450 3656 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0425 19:00:16.711464 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0425 19:00:16.711472 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0425 19:00:16.711916 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0425 19:00:16.711933 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0425 19:00:16.711941 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0425 19:00:16.712428 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0425 19:00:16.712444 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0425 19:00:16.712451 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0425 19:00:16.712888 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0425 19:00:16.712903 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0425 19:00:16.712909 3656 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0425 19:00:16.713169 3656 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0425 19:00:16.713183 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0425 19:00:16.713203 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0425 19:00:16.713835 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0425 19:00:16.713853 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0425 19:00:16.713860 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0425 19:00:16.714391 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0425 19:00:16.714416 3656 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0425 19:00:16.714429 3656 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0425 19:00:16.715428 3656 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0425 19:00:16.715448 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0425 19:00:16.715457 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0425 19:00:16.716688 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0425 19:00:16.716708 3656 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0425 19:00:16.716718 3656 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0425 19:00:16.717943 3656 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0425 19:00:16.717961 3656 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0425 19:00:16.717971 3656 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0425 19:00:16.719361 3656 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0425 19:00:16.719381 3656 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0425 19:00:16.719401 3656 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0425 19:00:16.720570 3656 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0425 19:00:16.725434 3656 egressfirewall.go:374] Starting EgressFirewall controller I0425 19:00:16.725467 3656 controller.go:133] Adding controller egress-firewall-controller event handlers I0425 19:00:16.725470 3656 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0425 19:00:16.725498 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0425 19:00:16.725510 3656 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0425 19:00:16.725499 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0425 19:00:16.725517 3656 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0425 19:00:16.725778 3656 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0425 19:00:16.725790 3656 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0425 19:00:16.725795 3656 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0425 19:00:16.725803 3656 egressqos.go:193] Setting up event handlers for EgressQoS I0425 19:00:16.725813 3656 egressfirewall.go:723] Syncing node "ip-10-0-0-74.ec2.internal" for egress firewall I0425 19:00:16.725827 3656 egressfirewall.go:723] Syncing node "ip-10-0-1-130.ec2.internal" for egress firewall I0425 19:00:16.725831 3656 egressfirewall.go:723] Syncing node "ip-10-0-2-191.ec2.internal" for egress firewall I0425 19:00:16.725926 3656 egressqos.go:245] Starting EgressQoS Controller I0425 19:00:16.725937 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0425 19:00:16.725944 3656 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0425 19:00:16.725950 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0425 19:00:16.725957 3656 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0425 19:00:16.725965 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0425 19:00:16.725973 3656 shared_informer.go:356] "Caches are synced" controller="egressqos" I0425 19:00:16.725977 3656 egressqos.go:259] Repairing EgressQoSes I0425 19:00:16.725981 3656 egressqos.go:399] Starting repairing loop for egressqos I0425 19:00:16.726080 3656 egressqos.go:401] Finished repairing loop for egressqos: 98.315µs I0425 19:00:16.726101 3656 egressservice_zone.go:132] Setting up event handlers for Egress Services I0425 19:00:16.726222 3656 egressservice_zone.go:210] Starting Egress Services Controller I0425 19:00:16.726233 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0425 19:00:16.726239 3656 shared_informer.go:356] "Caches are synced" controller="egressservices" I0425 19:00:16.726244 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0425 19:00:16.726249 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0425 19:00:16.726255 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0425 19:00:16.726261 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0425 19:00:16.726271 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0425 19:00:16.726278 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0425 19:00:16.726282 3656 egressservice_zone.go:228] Repairing Egress Services I0425 19:00:16.726358 3656 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-130.ec2.internal I0425 19:00:16.726406 3656 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-130.ec2.internal : 49.687µs I0425 19:00:16.726567 3656 master_controller.go:88] Starting Admin Policy Based Route Controller I0425 19:00:16.726582 3656 external_controller.go:278] Starting Admin Policy Based Route Controller I0425 19:00:16.726636 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-130.ec2.internal I0425 19:00:16.726646 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-130.ec2.internal: 11.385µs I0425 19:00:16.726653 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-191.ec2.internal I0425 19:00:16.726657 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-191.ec2.internal: 4.254µs I0425 19:00:16.726675 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-74.ec2.internal I0425 19:00:16.726679 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-74.ec2.internal: 18.581µs I0425 19:00:16.726798 3656 default_network_controller.go:585] Completing all the Watchers took 308.721056ms I0425 19:00:16.726816 3656 default_network_controller.go:589] Starting unidling controllers I0425 19:00:16.726828 3656 unidle.go:46] Registering OVN SB ControllerEvent handler I0425 19:00:16.726837 3656 unidle.go:63] Populating Initial ContollerEvent events I0425 19:00:16.726851 3656 unidle.go:79] Setting up event handlers for services I0425 19:00:16.726900 3656 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0425 19:00:16.789427 3656 default_node_network_controller.go:519] Node connection status = connected I0425 19:00:17.290193 3656 default_node_network_controller.go:519] Node connection status = connected I0425 19:00:17.790395 3656 default_node_network_controller.go:519] Node connection status = connected I0425 19:00:18.290128 3656 default_node_network_controller.go:519] Node connection status = connected I0425 19:00:18.523074 3656 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0425 19:00:18.790624 3656 default_node_network_controller.go:519] Node connection status = connected I0425 19:00:18.808274 3656 bridgeconfig.go:537] Gateway is ready I0425 19:00:18.808294 3656 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0425 19:00:18.820211 3656 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.130. Updating OVN encap IP. I0425 19:00:18.825676 3656 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.130 - it is already configured I0425 19:00:18.825689 3656 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.130/24:{} 172.20.0.1/32:{}]. Updating annotations. I0425 19:00:18.825913 3656 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.130/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-130.ec2.internal","mac-address":"0a:ff:cc:8f:5b:35","ip-addresses":["10.0.1.130/24"],"ip-address":"10.0.1.130/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:f0ce6ac8-41d6-4462-9c7a-946ed0deee40 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.130/24"}] on node ip-10-0-1-130.ec2.internal I0425 19:00:18.843425 3656 egressfirewall.go:723] Syncing node "ip-10-0-1-130.ec2.internal" for egress firewall I0425 19:00:18.843434 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-130.ec2.internal I0425 19:00:18.843455 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-130.ec2.internal: 27.667µs I0425 19:00:18.843485 3656 master.go:548] Adding or Updating local node "ip-10-0-1-130.ec2.internal" for network "default" I0425 19:00:18.843504 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-130.ec2.internal in network "default" I0425 19:00:18.843571 3656 node_tracker.go:169] Node ip-10-0-1-130.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:18.843589 3656 services_controller.go:574] Full service sync requested for network=default I0425 19:00:18.844267 3656 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.1.130 I0425 19:00:18.844499 3656 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-130.ec2.internal I0425 19:00:18.847968 3656 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-130.ec2.internal. Updating no re-route policies I0425 19:00:18.848054 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-dhbrt I0425 19:00:18.848067 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-rj8j4 I0425 19:00:18.848069 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-r9tw4 I0425 19:00:18.848075 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-dhbrt I0425 19:00:18.848078 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-rj8j4 I0425 19:00:18.848082 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-r9tw4 I0425 19:00:18.848097 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-ns76c I0425 19:00:18.848109 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-dhbrt after 0 failed attempt(s) I0425 19:00:18.848109 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-130.ec2.internal I0425 19:00:18.848114 3656 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-rj8j4, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-rj8j4: suppressed error logged: pod openshift-multus/network-metrics-daemon-rj8j4: no pod IPs found I0425 19:00:18.848058 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-cxcq7 I0425 19:00:18.848124 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-130.ec2.internal I0425 19:00:18.848121 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-sp64n I0425 19:00:18.848129 3656 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-9th9m I0425 19:00:18.848130 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-sldhs I0425 19:00:18.848132 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-cxcq7 I0425 19:00:18.848139 3656 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-9th9m I0425 19:00:18.848141 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-sp64n I0425 19:00:18.848143 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-sldhs I0425 19:00:18.848143 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-jszsp I0425 19:00:18.848152 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-130.ec2.internal after 0 failed attempt(s) I0425 19:00:18.848158 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-jszsp I0425 19:00:18.848125 3656 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-r9tw4, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-r9tw4: suppressed error logged: pod openshift-multus/network-metrics-daemon-r9tw4: no pod IPs found I0425 19:00:18.848161 3656 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-130.ec2.internal I0425 19:00:18.848170 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-sldhs after 0 failed attempt(s) I0425 19:00:18.848172 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-hblkv I0425 19:00:18.848169 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-2qtwp I0425 19:00:18.848174 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-wkqgf I0425 19:00:18.848180 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-hblkv I0425 19:00:18.848182 3656 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-130.ec2.internal I0425 19:00:18.848184 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-hcxrp I0425 19:00:18.848187 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-2qtwp I0425 19:00:18.848187 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-wkqgf I0425 19:00:18.848193 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-hcxrp I0425 19:00:18.848205 3656 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-130.ec2.internal after 0 failed attempt(s) I0425 19:00:18.848208 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-hcxrp after 0 failed attempt(s) I0425 19:00:18.848211 3656 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-hblkv] creating logical port openshift-network-diagnostics_network-check-target-hblkv for pod on switch ip-10-0-1-130.ec2.internal I0425 19:00:18.848156 3656 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-9th9m after 0 failed attempt(s) I0425 19:00:18.848159 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-cxcq7 after 0 failed attempt(s) I0425 19:00:18.848116 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-ns76c I0425 19:00:18.848239 3656 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-2qtwp, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-2qtwp: suppressed error logged: pod openshift-network-diagnostics/network-check-target-2qtwp: no pod IPs found I0425 19:00:18.848258 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-ns76c after 0 failed attempt(s) I0425 19:00:18.848165 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-s74ds I0425 19:00:18.848274 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-s74ds I0425 19:00:18.848159 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-sp64n after 0 failed attempt(s) I0425 19:00:18.848286 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-s74ds after 0 failed attempt(s) I0425 19:00:18.848289 3656 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-gwq25 I0425 19:00:18.848300 3656 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-gwq25 I0425 19:00:18.848312 3656 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-jszsp] creating logical port openshift-multus_network-metrics-daemon-jszsp for pod on switch ip-10-0-1-130.ec2.internal I0425 19:00:18.848345 3656 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-gwq25, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-gwq25: suppressed error logged: pod openshift-network-diagnostics/network-check-target-gwq25: no pod IPs found I0425 19:00:18.848350 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-wkqgf after 0 failed attempt(s) I0425 19:00:18.849167 3656 pods.go:271] [openshift-network-diagnostics/network-check-target-hblkv] addLogicalPort took 964.231µs, libovsdb time 345.938µs I0425 19:00:18.849184 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-hblkv after 0 failed attempt(s) I0425 19:00:18.849256 3656 pods.go:271] [openshift-multus/network-metrics-daemon-jszsp] addLogicalPort took 949.395µs, libovsdb time 303.664µs I0425 19:00:18.849266 3656 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-jszsp after 0 failed attempt(s) I0425 19:00:18.861173 3656 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0425 19:00:18.873190 3656 iptables.go:108] Creating table: mangle chain: OUTPUT I0425 19:00:18.876336 3656 iptables.go:108] Creating table: nat chain: OUTPUT I0425 19:00:18.911387 3656 iptables.go:108] Creating table: nat chain: PREROUTING I0425 19:00:18.959884 3656 iptables.go:108] Creating table: filter chain: FORWARD I0425 19:00:18.976340 3656 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0425 19:00:18.976449 3656 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0425 19:00:18.976498 3656 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0425 19:00:18.976574 3656 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0425 19:00:18.976635 3656 gateway_init.go:325] Gateway and management port readiness took 2.691366877s I0425 19:00:18.976664 3656 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0425 19:00:18.976702 3656 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0425 19:00:18.976714 3656 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0425 19:00:18.976723 3656 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0425 19:00:18.976792 3656 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0425 19:00:18.976886 3656 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 63.791µs I0425 19:00:18.977129 3656 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0425 19:00:18.977155 3656 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.513µs I0425 19:00:18.977163 3656 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 35.474µs I0425 19:00:18.977169 3656 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0425 19:00:18.977179 3656 nad_controller.go:379] [node-nad-controller NAD controller]: started I0425 19:00:18.977184 3656 default_node_network_controller.go:899] Starting the default node network controller I0425 19:00:18.977189 3656 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0425 19:00:18.977263 3656 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 12.904µs I0425 19:00:18.977380 3656 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0425 19:00:18.977388 3656 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0425 19:00:18.977410 3656 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c6c110 0xc0002eda40 0xc0002edab0 0xc0002edb20 0xc0002edb90 0xc0002edc00 0xc0002edce0 0xc000c7e3c0 0xc0002edd50 0xc0002ede30 0xc000c7e550 0xc0065ef340 0xc0002edea0 map[0x2ff9080:0xc00299cf60 0x2ff95c0:0xc00299d0e0 0x2ffa2e0:0xc002d0b200 0x2ffa580:0xc00299cf00 0x2ffad60:0xc00299cfc0 0x2ffb000:0xc00299d020 0x2ffb2a0:0xc00299d080 0x2ffba80:0xc00299d140 0x2ffbd20:0xc00299d1a0 0x3026980:0xc000e06c00 0x3026d20:0xc000e06c60 0x3027800:0xc000c4d380 0x3029160:0xc001c64000 0x30298a0:0xc000e07380 0x302a720:0xc001801860] 0xc00016ab60 0} I0425 19:00:18.977478 3656 gateway.go:151] Starting gateway service sync I0425 19:00:18.977890 3656 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0425 19:00:18.979980 3656 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0425 19:00:18.997633 3656 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0425 19:00:18.998913 3656 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0425 19:00:19.000093 3656 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0425 19:00:19.034623 3656 gateway.go:168] Gateway service sync done. Time taken: 57.131735ms I0425 19:00:19.035274 3656 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30525 I0425 19:00:19.035490 3656 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c6c110 0xc0002eda40 0xc0002edab0 0xc0002edb20 0xc0002edb90 0xc0002edc00 0xc0002edce0 0xc000c7e3c0 0xc0002edd50 0xc0002ede30 0xc000c7e550 0xc0065ef340 0xc0002edea0 map[0x2ff9080:0xc00299cf60 0x2ff95c0:0xc00299d0e0 0x2ffa2e0:0xc002d0b200 0x2ffa580:0xc00299cf00 0x2ffad60:0xc00299cfc0 0x2ffb000:0xc00299d020 0x2ffb2a0:0xc00299d080 0x2ffba80:0xc00299d140 0x2ffbd20:0xc00299d1a0 0x3026980:0xc000e06c00 0x3026d20:0xc000e06c60 0x3027800:0xc000c4d380 0x3029160:0xc001c64000 0x30298a0:0xc000e07380 0x302a720:0xc001801860] 0xc00016ab60 0} I0425 19:00:19.035965 3656 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000c6c110 0xc0002eda40 0xc0002edab0 0xc0002edb20 0xc0002edb90 0xc0002edc00 0xc0002edce0 0xc000c7e3c0 0xc0002edd50 0xc0002ede30 0xc000c7e550 0xc0065ef340 0xc0002edea0 map[0x2ff9080:0xc00299cf60 0x2ff95c0:0xc00299d0e0 0x2ffa2e0:0xc002d0b200 0x2ffa580:0xc00299cf00 0x2ffad60:0xc00299cfc0 0x2ffb000:0xc00299d020 0x2ffb2a0:0xc00299d080 0x2ffba80:0xc00299d140 0x2ffbd20:0xc00299d1a0 0x3026980:0xc000e06c00 0x3026d20:0xc000e06c60 0x3027800:0xc000c4d380 0x3029160:0xc001c64000 0x30298a0:0xc000e07380 0x302a720:0xc001801860] 0xc00016ab60 0} I0425 19:00:19.036015 3656 gateway.go:532] Reconciling gateway with updates I0425 19:00:19.041746 3656 gateway.go:339] Spawning Conntrack Rule Check Thread I0425 19:00:19.041830 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0425 19:00:19.041844 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0425 19:00:19.041850 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0425 19:00:19.041858 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0425 19:00:19.041858 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0425 19:00:19.041854 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0425 19:00:19.041866 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0425 19:00:19.041873 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0425 19:00:19.041873 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0425 19:00:19.041878 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0425 19:00:19.041881 3656 port_claim.go:71] Svc openshift-ingress/router-default: port 31078 is already open, no action needed I0425 19:00:19.041884 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0425 19:00:19.041888 3656 port_claim.go:71] Svc openshift-ingress/router-default: port 32489 is already open, no action needed I0425 19:00:19.041895 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0425 19:00:19.041849 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0425 19:00:19.041907 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0425 19:00:19.041916 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0425 19:00:19.041919 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0425 19:00:19.041933 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0425 19:00:19.041938 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0425 19:00:19.041940 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0425 19:00:19.041947 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0425 19:00:19.041956 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0425 19:00:19.041924 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0425 19:00:19.041960 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0425 19:00:19.041968 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0425 19:00:19.041975 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0425 19:00:19.041970 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0425 19:00:19.041981 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0425 19:00:19.041988 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0425 19:00:19.041986 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0425 19:00:19.042004 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0425 19:00:19.042005 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0425 19:00:19.042008 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0425 19:00:19.042004 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0425 19:00:19.041992 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0425 19:00:19.042015 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0425 19:00:19.042015 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0425 19:00:19.041990 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0425 19:00:19.042020 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0425 19:00:19.042029 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0425 19:00:19.042052 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0425 19:00:19.042056 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0425 19:00:19.042059 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0425 19:00:19.042064 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0425 19:00:19.042063 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0425 19:00:19.042066 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0425 19:00:19.042070 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0425 19:00:19.042077 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0425 19:00:19.042079 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0425 19:00:19.042051 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 19:00:19.042093 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 19:00:19.042062 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0425 19:00:19.042113 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0425 19:00:19.042119 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0425 19:00:19.042126 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0425 19:00:19.042132 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0425 19:00:19.042067 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0425 19:00:19.041863 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0425 19:00:19.042032 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0425 19:00:19.042179 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0425 19:00:19.042003 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0425 19:00:19.042211 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0425 19:00:19.041833 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0425 19:00:19.042220 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0425 19:00:19.042223 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0425 19:00:19.042267 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0425 19:00:19.042279 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0425 19:00:19.041848 3656 default_node_network_controller.go:944] Gateway and management port readiness took 64.452207ms I0425 19:00:19.042325 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0425 19:00:19.042334 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0425 19:00:19.042021 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0425 19:00:19.042024 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0425 19:00:19.042027 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0425 19:00:19.042384 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0425 19:00:19.042391 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0425 19:00:19.042389 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0425 19:00:19.042030 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0425 19:00:19.042414 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0425 19:00:19.042033 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0425 19:00:19.042016 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0425 19:00:19.042429 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0425 19:00:19.042434 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0425 19:00:19.042435 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0425 19:00:19.042058 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0425 19:00:19.042452 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0425 19:00:19.042454 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0425 19:00:19.042459 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0425 19:00:19.042466 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0425 19:00:19.042009 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0425 19:00:19.042076 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0425 19:00:19.042485 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0425 19:00:19.042492 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0425 19:00:19.042498 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0425 19:00:19.042150 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0425 19:00:19.042216 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0425 19:00:19.042600 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0425 19:00:19.042021 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0425 19:00:19.042403 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0425 19:00:19.042620 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0425 19:00:19.042421 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0425 19:00:19.042421 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0425 19:00:19.042707 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0425 19:00:19.042443 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0425 19:00:19.042476 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0425 19:00:19.042719 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0425 19:00:19.041865 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0425 19:00:19.043171 3656 node_ip_handler_linux.go:160] Node IP manager is running W0425 19:00:19.080501 3656 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0425 19:00:19.080728 3656 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0425 19:00:19.080892 3656 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0425 19:00:19.088457 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0425 19:00:19.088471 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0425 19:00:19.088489 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0425 19:00:19.128684 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0425 19:00:19.128718 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0425 19:00:19.128725 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0425 19:00:19.128732 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0425 19:00:19.128740 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0425 19:00:19.128748 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0425 19:00:19.128762 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0425 19:00:19.128774 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0425 19:00:19.128781 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0425 19:00:19.128786 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0425 19:00:19.128792 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0425 19:00:19.128797 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0425 19:00:19.128803 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0425 19:00:19.128810 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0425 19:00:19.168318 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0425 19:00:19.168344 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0425 19:00:19.168354 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0425 19:00:19.168362 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0425 19:00:19.168369 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0425 19:00:19.168375 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0425 19:00:19.168381 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0425 19:00:19.168387 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0425 19:00:19.168393 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0425 19:00:19.168399 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0425 19:00:19.168407 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0425 19:00:20.180710 3656 default_node_network_controller.go:1380] Starting node controller node sync W0425 19:00:20.180711 3656 egressip_healthcheck.go:76] Health checking using insecure connection I0425 19:00:20.180836 3656 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0425 19:00:20.193876 3656 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 13.14537ms I0425 19:00:20.194069 3656 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0425 19:00:20.194294 3656 egressservice_node.go:93] Setting up event handlers for Egress Services I0425 19:00:20.194455 3656 egressservice_node.go:183] Starting Egress Services Controller I0425 19:00:20.194494 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0425 19:00:20.194507 3656 shared_informer.go:356] "Caches are synced" controller="egressservices" I0425 19:00:20.194562 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0425 19:00:20.194569 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0425 19:00:20.194575 3656 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0425 19:00:20.194579 3656 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0425 19:00:20.194583 3656 egressservice_node.go:197] Repairing Egress Services I0425 19:00:20.205055 3656 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0425 19:00:20.205069 3656 external_controller.go:278] Starting Admin Policy Based Route Controller I0425 19:00:20.205341 3656 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0425 19:00:20.205352 3656 link_network_manager.go:119] Link manager is running I0425 19:00:20.205361 3656 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0425 19:00:20.205551 3656 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0425 19:00:20.205783 3656 vrf_manager.go:128] VRF manager is running I0425 19:00:20.205885 3656 node_controller_manager.go:437] Removing flows to drop GARP I0425 19:00:20.205899 3656 gateway.go:532] Reconciling gateway with updates I0425 19:00:20.220061 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0425 19:00:20.220071 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0425 19:00:20.220086 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0425 19:00:20.220082 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0425 19:00:20.220099 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0425 19:00:20.220110 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0425 19:00:20.220121 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0425 19:00:20.220127 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0425 19:00:20.220129 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0425 19:00:20.220135 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0425 19:00:20.220140 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0425 19:00:20.220145 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0425 19:00:20.220152 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0425 19:00:20.220167 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0425 19:00:20.220105 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0425 19:00:20.220184 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0425 19:00:20.220188 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0425 19:00:20.220213 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0425 19:00:20.220220 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0425 19:00:20.220212 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0425 19:00:20.220228 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0425 19:00:20.220126 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0425 19:00:20.220238 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0425 19:00:20.220240 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0425 19:00:20.220218 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0425 19:00:20.220248 3656 port_claim.go:71] Svc openshift-ingress/router-default: port 31078 is already open, no action needed I0425 19:00:20.220251 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0425 19:00:20.220254 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0425 19:00:20.220252 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0425 19:00:20.220264 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0425 19:00:20.220269 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0425 19:00:20.220194 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0425 19:00:20.220214 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 19:00:20.220282 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0425 19:00:20.220293 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 19:00:20.220293 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0425 19:00:20.220300 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0425 19:00:20.220312 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0425 19:00:20.220312 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0425 19:00:20.220314 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0425 19:00:20.220324 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0425 19:00:20.220329 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0425 19:00:20.220331 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0425 19:00:20.220330 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0425 19:00:20.220344 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0425 19:00:20.220349 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0425 19:00:20.220335 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0425 19:00:20.220297 3656 port_claim.go:71] Svc openshift-ingress/router-default: port 32489 is already open, no action needed I0425 19:00:20.220366 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0425 19:00:20.220376 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0425 19:00:20.220229 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0425 19:00:20.220379 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0425 19:00:20.220388 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0425 19:00:20.220392 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0425 19:00:20.220395 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0425 19:00:20.220229 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0425 19:00:20.220399 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0425 19:00:20.220401 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0425 19:00:20.220410 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0425 19:00:20.220411 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0425 19:00:20.220412 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0425 19:00:20.220415 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0425 19:00:20.220313 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0425 19:00:20.220423 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0425 19:00:20.220426 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0425 19:00:20.220389 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0425 19:00:20.220410 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0425 19:00:20.220392 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0425 19:00:20.220451 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0425 19:00:20.220240 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0425 19:00:20.220458 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0425 19:00:20.220468 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0425 19:00:20.220199 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0425 19:00:20.220483 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0425 19:00:20.220269 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0425 19:00:20.220090 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0425 19:00:20.220497 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0425 19:00:20.220503 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0425 19:00:20.220294 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0425 19:00:20.220223 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0425 19:00:20.220312 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0425 19:00:20.220523 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0425 19:00:20.220538 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0425 19:00:20.220555 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0425 19:00:20.220562 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0425 19:00:20.220575 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0425 19:00:20.220352 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0425 19:00:20.220584 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0425 19:00:20.220598 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0425 19:00:20.220592 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0425 19:00:20.220232 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0425 19:00:20.220222 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0425 19:00:20.220745 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0425 19:00:20.220764 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0425 19:00:20.220771 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0425 19:00:20.220418 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0425 19:00:20.220314 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0425 19:00:20.220814 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0425 19:00:20.220200 3656 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0425 19:00:20.220884 3656 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0425 19:00:20.250210 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0425 19:00:20.250239 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0425 19:00:20.250245 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0425 19:00:20.250251 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0425 19:00:20.250260 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0425 19:00:20.250267 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0425 19:00:20.250273 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0425 19:00:20.250278 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0425 19:00:20.250287 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0425 19:00:20.250292 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0425 19:00:20.295221 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0425 19:00:20.295232 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0425 19:00:20.295263 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0425 19:00:20.295270 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0425 19:00:20.295275 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0425 19:00:20.295282 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0425 19:00:20.295308 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0425 19:00:20.295317 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0425 19:00:20.295322 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0425 19:00:20.339679 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0425 19:00:20.339712 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0425 19:00:20.339723 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0425 19:00:20.339731 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0425 19:00:20.339737 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0425 19:00:20.339742 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0425 19:00:20.339748 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0425 19:00:20.339753 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0425 19:00:20.339759 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0425 19:00:20.339765 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0425 19:00:20.339771 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0425 19:00:20.339778 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0425 19:00:20.339789 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0425 19:00:20.339809 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0425 19:00:20.339826 3656 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0425 19:00:20.728952 3656 default_network_controller.go:1025] Node "ip-10-0-0-74.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:22.473585 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-130.ec2.internal I0425 19:00:22.473624 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-130.ec2.internal: 48.7µs I0425 19:00:22.500077 3656 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5c56ffbfbd-25zkt] creating logical port openshift-image-registry_image-registry-5c56ffbfbd-25zkt for pod on switch ip-10-0-1-130.ec2.internal I0425 19:00:22.500229 3656 kube.go:256] Updating pod openshift-image-registry/image-registry-5c56ffbfbd-25zkt I0425 19:00:22.524476 3656 pod.go:62] [openshift-image-registry/image-registry-5c56ffbfbd-25zkt] pod update took 24.282634ms I0425 19:00:22.524505 3656 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-5c56ffbfbd-25zkt] addLogicalPort annotation time took 24.317342ms I0425 19:00:22.525706 3656 pods.go:271] [openshift-image-registry/image-registry-5c56ffbfbd-25zkt] addLogicalPort took 25.665157ms, libovsdb time 948.696µs I0425 19:00:22.539880 3656 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-rl6sj] creating logical port openshift-dns_dns-default-rl6sj for pod on switch ip-10-0-1-130.ec2.internal I0425 19:00:22.539994 3656 kube.go:256] Updating pod openshift-dns/dns-default-rl6sj I0425 19:00:22.557576 3656 pod.go:62] [openshift-dns/dns-default-rl6sj] pod update took 17.602427ms I0425 19:00:22.557666 3656 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-rl6sj] addLogicalPort annotation time took 17.691404ms I0425 19:00:22.558971 3656 pods.go:271] [openshift-dns/dns-default-rl6sj] addLogicalPort took 19.100316ms, libovsdb time 992.99µs I0425 19:00:22.569569 3656 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-xth8b] creating logical port openshift-ingress-canary_ingress-canary-xth8b for pod on switch ip-10-0-1-130.ec2.internal I0425 19:00:22.569697 3656 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-xth8b I0425 19:00:22.576368 3656 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29619060-6l2v9 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:00:22.576492 3656 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 I0425 19:00:22.576747 3656 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-ncwt8] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-ncwt8 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:00:22.576905 3656 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-ncwt8 I0425 19:00:22.585219 3656 pod.go:62] [openshift-ingress-canary/ingress-canary-xth8b] pod update took 15.553252ms I0425 19:00:22.585245 3656 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-xth8b] addLogicalPort annotation time took 15.585613ms I0425 19:00:22.586678 3656 pods.go:271] [openshift-ingress-canary/ingress-canary-xth8b] addLogicalPort took 17.126684ms, libovsdb time 1.115713ms I0425 19:00:22.594689 3656 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9] pod update took 18.218344ms I0425 19:00:22.594719 3656 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9] addLogicalPort annotation time took 18.246526ms I0425 19:00:22.596106 3656 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9] addLogicalPort took 19.759462ms, libovsdb time 1.010698ms I0425 19:00:22.598845 3656 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-ncwt8] pod update took 21.980983ms I0425 19:00:22.598906 3656 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-ncwt8] addLogicalPort annotation time took 22.042346ms I0425 19:00:22.600203 3656 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-ncwt8] addLogicalPort took 23.469466ms, libovsdb time 1.032681ms I0425 19:00:23.564881 3656 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 d18e2c6bcebcede17e0cbaa7d7c81e6f828809e52d39dd7e04c9b188a0ca3e9c network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 d18e2c6bcebcede17e0cbaa7d7c81e6f828809e52d39dd7e04c9b188a0ca3e9c network default NAD default NAD key ] I0425 19:00:23.567779 3656 cni.go:452] [openshift-ingress-canary/ingress-canary-xth8b cd23301cbb0e934c6e153ae2354c050564016fecb0211d463340091a5a3cd93c network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-xth8b cd23301cbb0e934c6e153ae2354c050564016fecb0211d463340091a5a3cd93c network default NAD default NAD key ] I0425 19:00:23.568571 3656 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-ncwt8 26529dbf5a0842b3b465ea0b2b02d062f796cef75c101427cbccff1ef28e6b67 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-ncwt8 26529dbf5a0842b3b465ea0b2b02d062f796cef75c101427cbccff1ef28e6b67 network default NAD default NAD key ] I0425 19:00:23.608247 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-ncwt8, hostIfaceName: 26529dbf5a0842b, network: default, NAD default, SandboxID: "26529dbf5a0842b3b465ea0b2b02d062f796cef75c101427cbccff1ef28e6b67", PCI device ID: , UID: "e3a61021-9d28-48df-870a-057d19e384e1", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0425 19:00:23.609009 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29619060-6l2v9, hostIfaceName: d18e2c6bcebcede, network: default, NAD default, SandboxID: "d18e2c6bcebcede17e0cbaa7d7c81e6f828809e52d39dd7e04c9b188a0ca3e9c", PCI device ID: , UID: "da48131b-8514-42c7-a56a-16dc5c355b90", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0425 19:00:23.612100 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-xth8b, hostIfaceName: cd23301cbb0e934, network: default, NAD default, SandboxID: "cd23301cbb0e934c6e153ae2354c050564016fecb0211d463340091a5a3cd93c", PCI device ID: , UID: "0e0b228d-9c02-4892-99c2-4e471804589b", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0425 19:00:23.662590 3656 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-ncwt8 26529dbf5a0842b3b465ea0b2b02d062f796cef75c101427cbccff1ef28e6b67 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-ncwt8 26529dbf5a0842b3b465ea0b2b02d062f796cef75c101427cbccff1ef28e6b67 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:2f:e2:16:ff:e6\",\"name\":\"26529dbf5a0842b\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ca47f20b-b604-46da-b292-bef96a4ceb5a\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:00:23.665259 3656 cni.go:473] [openshift-ingress-canary/ingress-canary-xth8b cd23301cbb0e934c6e153ae2354c050564016fecb0211d463340091a5a3cd93c network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-xth8b cd23301cbb0e934c6e153ae2354c050564016fecb0211d463340091a5a3cd93c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:38:89:29:1d:16\",\"name\":\"cd23301cbb0e934\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/870a2669-0536-4fa3-bbc6-9705bb69656e\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:00:23.670122 3656 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 d18e2c6bcebcede17e0cbaa7d7c81e6f828809e52d39dd7e04c9b188a0ca3e9c network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 d18e2c6bcebcede17e0cbaa7d7c81e6f828809e52d39dd7e04c9b188a0ca3e9c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:ce:a5:74:3a:d3\",\"name\":\"d18e2c6bcebcede\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6708e5f2-5843-45b5-a3d1-4b6ff89df325\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:00:24.742848 3656 default_network_controller.go:1025] Node "ip-10-0-2-191.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:26.593109 3656 cni.go:452] [openshift-network-diagnostics/network-check-target-hblkv d1d56356d5f7b09fcf6457c65e1f8ce1f7f121c6e4ee3a89df93c44eb3b3201d network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-hblkv d1d56356d5f7b09fcf6457c65e1f8ce1f7f121c6e4ee3a89df93c44eb3b3201d network default NAD default NAD key ] I0425 19:00:26.603474 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-hblkv, hostIfaceName: d1d56356d5f7b09, network: default, NAD default, SandboxID: "d1d56356d5f7b09fcf6457c65e1f8ce1f7f121c6e4ee3a89df93c44eb3b3201d", PCI device ID: , UID: "41218227-49b4-450b-bfef-bd3ea0ba8637", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0425 19:00:26.652464 3656 cni.go:473] [openshift-network-diagnostics/network-check-target-hblkv d1d56356d5f7b09fcf6457c65e1f8ce1f7f121c6e4ee3a89df93c44eb3b3201d network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-hblkv d1d56356d5f7b09fcf6457c65e1f8ce1f7f121c6e4ee3a89df93c44eb3b3201d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:25:5e:b3:e8:a1\",\"name\":\"d1d56356d5f7b09\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/54f85027-9972-46ce-89c0-fef15153994a\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:00:29.424380 3656 egressfirewall.go:723] Syncing node "ip-10-0-0-74.ec2.internal" for egress firewall I0425 19:00:29.424453 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-74.ec2.internal I0425 19:00:29.424469 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-74.ec2.internal: 21.413µs I0425 19:00:29.424492 3656 default_network_controller.go:1025] Node "ip-10-0-0-74.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0425 19:00:31.792693 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0425 19:00:32.781779 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 19:00:33.531424 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-191.ec2.internal I0425 19:00:33.531445 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-191.ec2.internal: 29.922µs I0425 19:00:33.531466 3656 egressfirewall.go:723] Syncing node "ip-10-0-2-191.ec2.internal" for egress firewall I0425 19:00:33.531532 3656 default_network_controller.go:1025] Node "ip-10-0-2-191.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:34.062827 3656 default_network_controller.go:1025] Node "ip-10-0-0-74.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:34.077826 3656 default_network_controller.go:1025] Node "ip-10-0-2-191.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:34.213291 3656 default_network_controller.go:1025] Node "ip-10-0-0-74.ec2.internal" in remote zone "ip-10-0-0-74.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:34.213325 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-74.ec2.internal in network "default" I0425 19:00:34.213368 3656 node_tracker.go:235] Node ip-10-0-0-74.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-74.ec2.internal" I0425 19:00:34.213382 3656 node_tracker.go:169] Node ip-10-0-0-74.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:34.213397 3656 services_controller.go:574] Full service sync requested for network=default W0425 19:00:34.213670 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0425 19:00:34.213858 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 19:00:34.288490 3656 default_network_controller.go:1025] Node "ip-10-0-0-74.ec2.internal" in remote zone "ip-10-0-0-74.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:34.349805 3656 default_network_controller.go:1025] Node "ip-10-0-0-74.ec2.internal" in remote zone "ip-10-0-0-74.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:34.349827 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-74.ec2.internal in network "default" I0425 19:00:34.349878 3656 node_tracker.go:169] Node ip-10-0-0-74.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:34.349893 3656 services_controller.go:574] Full service sync requested for network=default W0425 19:00:34.350115 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0425 19:00:34.350282 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 19:00:34.350819 3656 zone_ic_handler.go:195] Time taken to create transit switch: 281.201µs I0425 19:00:34.350833 3656 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-74.ec2.internal for the network default I0425 19:00:34.351759 3656 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-74.ec2.internal" on network "default" took: 1.234415ms I0425 19:00:34.937747 3656 default_network_controller.go:1025] Node "ip-10-0-2-191.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:36.421856 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-74.ec2.internal I0425 19:00:36.421882 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-74.ec2.internal: 36.198µs I0425 19:00:36.421887 3656 egressfirewall.go:723] Syncing node "ip-10-0-0-74.ec2.internal" for egress firewall I0425 19:00:36.421916 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-74.ec2.internal in network "default" I0425 19:00:36.421939 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-74.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0425 19:00:36.421948 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-74.ec2.internal", IP: 10.0.0.74 to PMTUD blocking rules I0425 19:00:36.421981 3656 node_tracker.go:169] Node ip-10-0-0-74.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:36.422002 3656 services_controller.go:574] Full service sync requested for network=default W0425 19:00:36.422358 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0425 19:00:36.422512 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0425 19:00:36.428014 3656 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-74.ec2.internal. Updating no re-route policies I0425 19:00:36.444982 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-74.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0425 19:00:36.444998 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-74.ec2.internal", IP: 10.0.0.74 to PMTUD blocking rules I0425 19:00:38.134543 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-191.ec2.internal in network "default" I0425 19:00:38.134573 3656 node_tracker.go:235] Node ip-10-0-2-191.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-191.ec2.internal" I0425 19:00:38.134590 3656 node_tracker.go:169] Node ip-10-0-2-191.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:38.134605 3656 services_controller.go:574] Full service sync requested for network=default I0425 19:00:38.134621 3656 default_network_controller.go:1025] Node "ip-10-0-2-191.ec2.internal" in remote zone "ip-10-0-2-191.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0425 19:00:38.134907 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0425 19:00:38.135086 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0425 19:00:38.171859 3656 default_network_controller.go:1025] Node "ip-10-0-2-191.ec2.internal" in remote zone "ip-10-0-2-191.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:38.275127 3656 default_network_controller.go:1025] Node "ip-10-0-2-191.ec2.internal" in remote zone "ip-10-0-2-191.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0425 19:00:38.275149 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-191.ec2.internal in network "default" I0425 19:00:38.275200 3656 node_tracker.go:169] Node ip-10-0-2-191.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:38.275214 3656 services_controller.go:574] Full service sync requested for network=default W0425 19:00:38.275585 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0425 19:00:38.275772 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0425 19:00:38.276215 3656 zone_ic_handler.go:195] Time taken to create transit switch: 334.684µs I0425 19:00:38.276227 3656 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-191.ec2.internal for the network default I0425 19:00:38.277267 3656 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-191.ec2.internal" on network "default" took: 1.404152ms I0425 19:00:40.214675 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-74.ec2.internal I0425 19:00:40.214712 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-74.ec2.internal: 45.31µs I0425 19:00:40.836980 3656 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-191.ec2.internal in network "default" I0425 19:00:40.837054 3656 node_tracker.go:169] Node ip-10-0-2-191.ec2.internal switch + router changed, syncing services in network "default" I0425 19:00:40.837068 3656 services_controller.go:574] Full service sync requested for network=default I0425 19:00:40.837078 3656 egressfirewall.go:723] Syncing node "ip-10-0-2-191.ec2.internal" for egress firewall I0425 19:00:40.837098 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-191.ec2.internal I0425 19:00:40.837113 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-191.ec2.internal: 19.089µs I0425 19:00:40.837159 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-191.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0425 19:00:40.837177 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-191.ec2.internal", IP: 10.0.2.191 to PMTUD blocking rules W0425 19:00:40.837308 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0425 19:00:40.837551 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0425 19:00:40.843323 3656 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-191.ec2.internal. Updating no re-route policies I0425 19:00:40.861169 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-191.ec2.internal", IP: 10.0.2.191 to PMTUD blocking rules I0425 19:00:40.861188 3656 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-191.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0425 19:00:44.318389 3656 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-191.ec2.internal I0425 19:00:44.318427 3656 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-191.ec2.internal: 47.888µs W0425 19:00:44.804856 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0425 19:00:47.587535 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0425 19:00:52.681792 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0425 19:01:02.783651 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0425 19:01:28.821020 3656 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d7dgr6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:01:28.821561 3656 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d7dgr6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:01:28.821594 3656 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d7dgr6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:01:28.821604 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0425 19:01:28.821618 3656 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d7dgr6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:01:28.821623 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0425 19:01:29.851632 3656 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d7dgr6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:01:29.851663 3656 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d7dgr6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:01:29.851680 3656 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d7dgr6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:01:29.851695 3656 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d7dgr6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:01:32.924593 3656 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0425 19:01:32.935977 3656 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0425 19:02:35.880986 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0425 19:02:36.671599 3656 namespace.go:98] [openshift-service-ca] adding namespace I0425 19:02:36.672519 3656 namespace.go:102] [openshift-service-ca] adding namespace took 898.313µs I0425 19:02:36.717489 3656 namespace.go:142] [openshift-service-ca] updating namespace I0425 19:02:37.306807 3656 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0425 19:02:37.307677 3656 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 850.751µs I0425 19:02:37.333142 3656 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0425 19:02:38.397360 3656 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-lvn4x] creating logical port openshift-service-ca_service-ca-d9cb48497-lvn4x for pod on switch ip-10-0-1-130.ec2.internal I0425 19:02:38.397500 3656 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-lvn4x I0425 19:02:38.414433 3656 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-lvn4x] pod update took 16.965842ms I0425 19:02:38.414464 3656 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-lvn4x] addLogicalPort annotation time took 17.000564ms I0425 19:02:38.415580 3656 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-lvn4x] addLogicalPort took 18.241049ms, libovsdb time 846.794µs I0425 19:02:38.751150 3656 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-lvn4x 4ff76a32f3aa1c868fbe91cd2466c889e50107fc56bd22bac547e29f05c1cd08 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-lvn4x 4ff76a32f3aa1c868fbe91cd2466c889e50107fc56bd22bac547e29f05c1cd08 network default NAD default NAD key ] I0425 19:02:38.760812 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-lvn4x, hostIfaceName: 4ff76a32f3aa1c8, network: default, NAD default, SandboxID: "4ff76a32f3aa1c868fbe91cd2466c889e50107fc56bd22bac547e29f05c1cd08", PCI device ID: , UID: "a186d2ca-78a6-4bbd-8d1d-cd6b5cc2b5c4", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0425 19:02:38.806510 3656 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-lvn4x 4ff76a32f3aa1c868fbe91cd2466c889e50107fc56bd22bac547e29f05c1cd08 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-lvn4x 4ff76a32f3aa1c868fbe91cd2466c889e50107fc56bd22bac547e29f05c1cd08 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:2e:38:25:4e:f4\",\"name\":\"4ff76a32f3aa1c8\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/41118f24-3f91-4f6a-b772-7933a6b7fe39\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:02:46.957598 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0425 19:03:03.609141 3656 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0425 19:03:03.617507 3656 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0425 19:03:03.617530 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0425 19:03:03.617535 3656 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0425 19:03:03.668872 3656 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-pz6qw] creating logical port openshift-insights_insights-runtime-extractor-pz6qw for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:03.668964 3656 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-pz6qw W0425 19:03:03.693899 3656 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0425 19:03:03.697944 3656 pod.go:62] [openshift-insights/insights-runtime-extractor-pz6qw] pod update took 29.007137ms I0425 19:03:03.697963 3656 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-pz6qw] addLogicalPort annotation time took 29.029603ms I0425 19:03:03.699074 3656 pods.go:271] [openshift-insights/insights-runtime-extractor-pz6qw] addLogicalPort took 30.217938ms, libovsdb time 836.19µs I0425 19:03:04.025212 3656 cni.go:452] [openshift-insights/insights-runtime-extractor-pz6qw 2b278b72d034dfed763301a787661c11995193919a83349021bf4ae076c20e7c network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-pz6qw 2b278b72d034dfed763301a787661c11995193919a83349021bf4ae076c20e7c network default NAD default NAD key ] I0425 19:03:04.034291 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-pz6qw, hostIfaceName: 2b278b72d034dfe, network: default, NAD default, SandboxID: "2b278b72d034dfed763301a787661c11995193919a83349021bf4ae076c20e7c", PCI device ID: , UID: "6582958e-621c-404d-8e78-37b1e57cc24b", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0425 19:03:04.081253 3656 cni.go:473] [openshift-insights/insights-runtime-extractor-pz6qw 2b278b72d034dfed763301a787661c11995193919a83349021bf4ae076c20e7c network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-pz6qw 2b278b72d034dfed763301a787661c11995193919a83349021bf4ae076c20e7c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:9e:22:ee:db:cc\",\"name\":\"2b278b72d034dfe\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2826ce97-69f8-4570-a4c9-224af5364973\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:03:04.981658 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 19:03:04.981680 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 19:03:04.981689 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 19:03:04.983274 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0425 19:03:04.983289 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0425 19:03:05.006184 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0425 19:03:05.006199 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0425 19:03:05.009235 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 19:03:05.009253 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 19:03:05.009260 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 19:03:05.050097 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 19:03:05.050119 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 19:03:05.050125 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0425 19:03:05.053616 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0425 19:03:05.053630 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0425 19:03:05.144673 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5c56ffbfbd-25zkt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:03:05.144712 3656 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5c56ffbfbd-25zkt I0425 19:03:05.145909 3656 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5c56ffbfbd-25zkt, ips: 10.128.0.5 I0425 19:03:05.145933 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5c56ffbfbd-25zkt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:03:05.157425 3656 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7dd54dd4f9-6dbjq] creating logical port openshift-image-registry_image-registry-7dd54dd4f9-6dbjq for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:05.157531 3656 kube.go:256] Updating pod openshift-image-registry/image-registry-7dd54dd4f9-6dbjq I0425 19:03:05.168204 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5c56ffbfbd-25zkt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:03:05.168238 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5c56ffbfbd-25zkt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:03:05.172763 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5c56ffbfbd-25zkt I0425 19:03:05.172822 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5c56ffbfbd-25zkt I0425 19:03:05.177059 3656 pod.go:62] [openshift-image-registry/image-registry-7dd54dd4f9-6dbjq] pod update took 19.542643ms I0425 19:03:05.177078 3656 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7dd54dd4f9-6dbjq] addLogicalPort annotation time took 19.58545ms I0425 19:03:05.178239 3656 pods.go:271] [openshift-image-registry/image-registry-7dd54dd4f9-6dbjq] addLogicalPort took 20.835019ms, libovsdb time 872.686µs I0425 19:03:05.516192 3656 cni.go:452] [openshift-image-registry/image-registry-7dd54dd4f9-6dbjq 2b13f8ed0f1b09928e4a2a294d1e26d5e24dcd1d7c07c3eeb97435182324d7eb network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7dd54dd4f9-6dbjq 2b13f8ed0f1b09928e4a2a294d1e26d5e24dcd1d7c07c3eeb97435182324d7eb network default NAD default NAD key ] I0425 19:03:05.526310 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7dd54dd4f9-6dbjq, hostIfaceName: 2b13f8ed0f1b099, network: default, NAD default, SandboxID: "2b13f8ed0f1b09928e4a2a294d1e26d5e24dcd1d7c07c3eeb97435182324d7eb", PCI device ID: , UID: "02192b17-75f6-41bd-98f0-4e160278f3a0", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0425 19:03:05.575812 3656 cni.go:473] [openshift-image-registry/image-registry-7dd54dd4f9-6dbjq 2b13f8ed0f1b09928e4a2a294d1e26d5e24dcd1d7c07c3eeb97435182324d7eb network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7dd54dd4f9-6dbjq 2b13f8ed0f1b09928e4a2a294d1e26d5e24dcd1d7c07c3eeb97435182324d7eb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:be:fc:2c:52:e5\",\"name\":\"2b13f8ed0f1b099\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/121a6020-589b-4dd9-94c3-e64e5e91783d\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:03:06.154496 3656 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0425 19:03:07.539784 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-gj7nz] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-gj7nz for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:07.539898 3656 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-gj7nz I0425 19:03:07.554926 3656 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-gj7nz] pod update took 15.05618ms I0425 19:03:07.554947 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-gj7nz] addLogicalPort annotation time took 15.081687ms I0425 19:03:07.556094 3656 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-gj7nz] addLogicalPort took 16.327388ms, libovsdb time 846.579µs I0425 19:03:07.896121 3656 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-gj7nz 9b7a51a75869236191b1f3f0cd58c2d17f145e5a89dab98725254a4d95e51e8f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-gj7nz 9b7a51a75869236191b1f3f0cd58c2d17f145e5a89dab98725254a4d95e51e8f network default NAD default NAD key ] I0425 19:03:07.907347 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-gj7nz, hostIfaceName: 9b7a51a75869236, network: default, NAD default, SandboxID: "9b7a51a75869236191b1f3f0cd58c2d17f145e5a89dab98725254a4d95e51e8f", PCI device ID: , UID: "1a897f64-2dc9-47f4-9f74-631ecabbd56b", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0425 19:03:07.952833 3656 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-gj7nz 9b7a51a75869236191b1f3f0cd58c2d17f145e5a89dab98725254a4d95e51e8f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-gj7nz 9b7a51a75869236191b1f3f0cd58c2d17f145e5a89dab98725254a4d95e51e8f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:c5:65:80:d7:c7\",\"name\":\"9b7a51a75869236\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d24d1bc8-301c-4256-a43a-61a7419e81a3\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:03:10.042066 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0425 19:03:10.186813 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0425 19:03:10.647195 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-j6clh] creating logical port openshift-monitoring_prometheus-operator-76b589d668-j6clh for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:10.647287 3656 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-j6clh I0425 19:03:10.662344 3656 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-j6clh] pod update took 15.07674ms I0425 19:03:10.662373 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-j6clh] addLogicalPort annotation time took 15.112708ms I0425 19:03:10.663661 3656 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-j6clh] addLogicalPort took 16.481452ms, libovsdb time 909.04µs I0425 19:03:11.603085 3656 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-j6clh 3540405843ef8ded23969d0c7f5ff47510ccda4741e505aa193c4c64a2b0fb3f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-j6clh 3540405843ef8ded23969d0c7f5ff47510ccda4741e505aa193c4c64a2b0fb3f network default NAD default NAD key ] I0425 19:03:11.612967 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-j6clh, hostIfaceName: 3540405843ef8de, network: default, NAD default, SandboxID: "3540405843ef8ded23969d0c7f5ff47510ccda4741e505aa193c4c64a2b0fb3f", PCI device ID: , UID: "03255d2e-fb8f-4a20-aa13-375dee1373a0", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0425 19:03:11.660412 3656 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-j6clh 3540405843ef8ded23969d0c7f5ff47510ccda4741e505aa193c4c64a2b0fb3f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-j6clh 3540405843ef8ded23969d0c7f5ff47510ccda4741e505aa193c4c64a2b0fb3f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:fe:15:62:82:47\",\"name\":\"3540405843ef8de\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bc3a72ea-ff40-4763-b1d2-44a530e79ecb\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:03:13.720405 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0425 19:03:13.720429 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0425 19:03:16.283891 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-wpf9c] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-wpf9c for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:16.283991 3656 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-wpf9c I0425 19:03:16.307717 3656 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-wpf9c] pod update took 23.750564ms I0425 19:03:16.307740 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-wpf9c] addLogicalPort annotation time took 23.779565ms I0425 19:03:16.309019 3656 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-wpf9c] addLogicalPort took 25.146397ms, libovsdb time 979.056µs I0425 19:03:16.321142 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-jhx6s] creating logical port openshift-monitoring_kube-state-metrics-6dfffbb599-jhx6s for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:16.321243 3656 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-6dfffbb599-jhx6s I0425 19:03:16.343241 3656 pod.go:62] [openshift-monitoring/kube-state-metrics-6dfffbb599-jhx6s] pod update took 22.025921ms I0425 19:03:16.343264 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-jhx6s] addLogicalPort annotation time took 22.06231ms I0425 19:03:16.344787 3656 pods.go:271] [openshift-monitoring/kube-state-metrics-6dfffbb599-jhx6s] addLogicalPort took 23.657808ms, libovsdb time 1.060656ms I0425 19:03:16.639128 3656 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-wpf9c 6240dcda35af6fadd64c12328bca3c51f12220fefe388184ddb0bed0ff05ac5f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-wpf9c 6240dcda35af6fadd64c12328bca3c51f12220fefe388184ddb0bed0ff05ac5f network default NAD default NAD key ] I0425 19:03:16.649373 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-wpf9c, hostIfaceName: 6240dcda35af6fa, network: default, NAD default, SandboxID: "6240dcda35af6fadd64c12328bca3c51f12220fefe388184ddb0bed0ff05ac5f", PCI device ID: , UID: "f0f7ce60-ed0b-4ed9-a0cc-5d7ed0ec0e2e", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0425 19:03:16.697773 3656 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-wpf9c 6240dcda35af6fadd64c12328bca3c51f12220fefe388184ddb0bed0ff05ac5f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-wpf9c 6240dcda35af6fadd64c12328bca3c51f12220fefe388184ddb0bed0ff05ac5f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:63:1f:f2:a0:1a\",\"name\":\"6240dcda35af6fa\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2a0de024-af77-45bb-ad3f-5747caa8bea1\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:03:17.277538 3656 cni.go:452] [openshift-monitoring/kube-state-metrics-6dfffbb599-jhx6s 0b0dd7eed079082b14205db614883e07f8e38eb5acce5c6e849784111b43ac9a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-jhx6s 0b0dd7eed079082b14205db614883e07f8e38eb5acce5c6e849784111b43ac9a network default NAD default NAD key ] I0425 19:03:17.305452 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-6dfffbb599-jhx6s, hostIfaceName: 0b0dd7eed079082, network: default, NAD default, SandboxID: "0b0dd7eed079082b14205db614883e07f8e38eb5acce5c6e849784111b43ac9a", PCI device ID: , UID: "836c6cd9-db56-4694-b237-1894d26c4715", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0425 19:03:17.355404 3656 cni.go:473] [openshift-monitoring/kube-state-metrics-6dfffbb599-jhx6s 0b0dd7eed079082b14205db614883e07f8e38eb5acce5c6e849784111b43ac9a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-jhx6s 0b0dd7eed079082b14205db614883e07f8e38eb5acce5c6e849784111b43ac9a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:76:0a:5c:a8:2b\",\"name\":\"0b0dd7eed079082\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1edbd7a5-2be0-4627-ae4b-ce9a0c790975\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:03:17.384838 3656 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-dvzwd] creating logical port openshift-console_downloads-5c54f4944f-dvzwd for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:17.385089 3656 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-dvzwd I0425 19:03:17.412685 3656 pod.go:62] [openshift-console/downloads-5c54f4944f-dvzwd] pod update took 27.653194ms I0425 19:03:17.412720 3656 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-dvzwd] addLogicalPort annotation time took 27.693335ms I0425 19:03:17.414101 3656 pods.go:271] [openshift-console/downloads-5c54f4944f-dvzwd] addLogicalPort took 29.283965ms, libovsdb time 990.479µs I0425 19:03:17.834591 3656 cni.go:452] [openshift-console/downloads-5c54f4944f-dvzwd 4aeee88ba2cde28b65b7223a8d6c7a0e067121806c1cefa40cc6d81e3f93b18d network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-dvzwd 4aeee88ba2cde28b65b7223a8d6c7a0e067121806c1cefa40cc6d81e3f93b18d network default NAD default NAD key ] I0425 19:03:17.844796 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-dvzwd, hostIfaceName: 4aeee88ba2cde28, network: default, NAD default, SandboxID: "4aeee88ba2cde28b65b7223a8d6c7a0e067121806c1cefa40cc6d81e3f93b18d", PCI device ID: , UID: "e54899ae-0f84-433f-8176-af705c7f258b", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0425 19:03:17.897795 3656 cni.go:473] [openshift-console/downloads-5c54f4944f-dvzwd 4aeee88ba2cde28b65b7223a8d6c7a0e067121806c1cefa40cc6d81e3f93b18d network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-dvzwd 4aeee88ba2cde28b65b7223a8d6c7a0e067121806c1cefa40cc6d81e3f93b18d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:08:15:34:e9:9a\",\"name\":\"4aeee88ba2cde28\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/07bb2c55-786c-479a-9d0f-251a2edb90db\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:03:17.966712 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0425 19:03:17.966749 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0425 19:03:19.656988 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-d7f9bf64c-5mprr] creating logical port openshift-monitoring_thanos-querier-d7f9bf64c-5mprr for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:19.657220 3656 kube.go:256] Updating pod openshift-monitoring/thanos-querier-d7f9bf64c-5mprr I0425 19:03:19.683473 3656 pod.go:62] [openshift-monitoring/thanos-querier-d7f9bf64c-5mprr] pod update took 26.35352ms I0425 19:03:19.683499 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-d7f9bf64c-5mprr] addLogicalPort annotation time took 26.384414ms I0425 19:03:19.684774 3656 pods.go:271] [openshift-monitoring/thanos-querier-d7f9bf64c-5mprr] addLogicalPort took 27.805895ms, libovsdb time 927.206µs I0425 19:03:20.017154 3656 cni.go:452] [openshift-monitoring/thanos-querier-d7f9bf64c-5mprr 330825ba5dd385f766c2be274ec2a2836c1cf629e1c45ae5162f9c74b7f44efc network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-d7f9bf64c-5mprr 330825ba5dd385f766c2be274ec2a2836c1cf629e1c45ae5162f9c74b7f44efc network default NAD default NAD key ] I0425 19:03:20.026409 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-d7f9bf64c-5mprr, hostIfaceName: 330825ba5dd385f, network: default, NAD default, SandboxID: "330825ba5dd385f766c2be274ec2a2836c1cf629e1c45ae5162f9c74b7f44efc", PCI device ID: , UID: "472d53f5-bbf2-451e-ba80-cd75dd02a13a", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0425 19:03:20.077548 3656 cni.go:473] [openshift-monitoring/thanos-querier-d7f9bf64c-5mprr 330825ba5dd385f766c2be274ec2a2836c1cf629e1c45ae5162f9c74b7f44efc network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-d7f9bf64c-5mprr 330825ba5dd385f766c2be274ec2a2836c1cf629e1c45ae5162f9c74b7f44efc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:4e:16:4a:15:b2\",\"name\":\"330825ba5dd385f\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb3df53f-7d64-4e4c-925f-250ba54fd076\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:03:20.097533 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0425 19:03:20.097560 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0425 19:03:20.097566 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0425 19:03:20.902405 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-n2d55] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-n2d55 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:20.902530 3656 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-n2d55 I0425 19:03:20.919868 3656 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-n2d55] pod update took 17.381396ms I0425 19:03:20.919898 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-n2d55] addLogicalPort annotation time took 17.414616ms I0425 19:03:20.921109 3656 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-n2d55] addLogicalPort took 18.725294ms, libovsdb time 838.114µs W0425 19:03:21.109370 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0425 19:03:21.109390 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0425 19:03:21.109397 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0425 19:03:22.804897 3656 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-n2d55 9783303750d731b7e50d8c97f18e28deb1ccf564fda92c88c29fffa0fab2b72d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-n2d55 9783303750d731b7e50d8c97f18e28deb1ccf564fda92c88c29fffa0fab2b72d network default NAD default NAD key ] I0425 19:03:22.815951 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-n2d55, hostIfaceName: 9783303750d731b, network: default, NAD default, SandboxID: "9783303750d731b7e50d8c97f18e28deb1ccf564fda92c88c29fffa0fab2b72d", PCI device ID: , UID: "c4e9eb3d-6142-4a56-9aba-87d86fb06fe8", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0425 19:03:22.870723 3656 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-n2d55 9783303750d731b7e50d8c97f18e28deb1ccf564fda92c88c29fffa0fab2b72d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-n2d55 9783303750d731b7e50d8c97f18e28deb1ccf564fda92c88c29fffa0fab2b72d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:97:76:fa:c5:91\",\"name\":\"9783303750d731b\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ed3bbc66-8df5-4bbb-bb60-028a9db766b9\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:03:23.254601 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 19:03:23.254620 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 19:03:23.254624 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 19:03:23.254628 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0425 19:03:23.557951 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:23.558147 3656 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0425 19:03:23.591304 3656 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 33.207993ms I0425 19:03:23.591334 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 33.243678ms I0425 19:03:23.593057 3656 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 35.123489ms, libovsdb time 1.114882ms W0425 19:03:24.074127 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0425 19:03:24.111325 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0425 19:03:24.149661 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 19:03:24.149688 3656 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 19:03:24.149694 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 19:03:24.149700 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier I0425 19:03:24.232743 3656 cni.go:452] [openshift-monitoring/prometheus-k8s-1 ae84c0fa9c65465c8c37a0fd08a6d24ead97f447b41f902b6cffcec396ff9059 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 ae84c0fa9c65465c8c37a0fd08a6d24ead97f447b41f902b6cffcec396ff9059 network default NAD default NAD key ] I0425 19:03:24.243462 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: ae84c0fa9c65465, network: default, NAD default, SandboxID: "ae84c0fa9c65465c8c37a0fd08a6d24ead97f447b41f902b6cffcec396ff9059", PCI device ID: , UID: "7413c517-0195-4887-b7fe-3e96bb555227", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0425 19:03:24.300927 3656 cni.go:473] [openshift-monitoring/prometheus-k8s-1 ae84c0fa9c65465c8c37a0fd08a6d24ead97f447b41f902b6cffcec396ff9059 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 ae84c0fa9c65465c8c37a0fd08a6d24ead97f447b41f902b6cffcec396ff9059 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:35:1a:b8:c4:5a\",\"name\":\"ae84c0fa9c65465\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3da78d64-5282-43f3-bdfb-c49cd8e0e299\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:03:25.108519 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:03:25.108538 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:03:25.277092 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0425 19:03:26.271305 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:03:26.271324 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:03:27.161062 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0425 19:03:28.984567 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-58fdcd69f-xbx2c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:03:28.985189 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-58fdcd69f-xbx2c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:03:29.131833 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-58fdcd69f-xbx2c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:03:29.131860 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-58fdcd69f-xbx2c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:03:29.140760 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-58fdcd69f-xbx2c I0425 19:03:29.140792 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-58fdcd69f-xbx2c W0425 19:03:29.256058 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 19:03:29.256081 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 19:03:29.256086 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0425 19:03:29.256091 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0425 19:03:29.870138 3656 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0425 19:03:31.186070 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0425 19:03:35.078003 3656 base_network_controller_pods.go:487] [default/openshift-console/console-b6878fd46-8ftsx] creating logical port openshift-console_console-b6878fd46-8ftsx for pod on switch ip-10-0-1-130.ec2.internal I0425 19:03:35.078134 3656 kube.go:256] Updating pod openshift-console/console-b6878fd46-8ftsx I0425 19:03:35.107581 3656 pod.go:62] [openshift-console/console-b6878fd46-8ftsx] pod update took 29.471697ms I0425 19:03:35.107623 3656 base_network_controller_pods.go:951] [default/openshift-console/console-b6878fd46-8ftsx] addLogicalPort annotation time took 29.515126ms I0425 19:03:35.109115 3656 pods.go:271] [openshift-console/console-b6878fd46-8ftsx] addLogicalPort took 31.132726ms, libovsdb time 1.034538ms I0425 19:03:35.451490 3656 cni.go:452] [openshift-console/console-b6878fd46-8ftsx 9a476b33091cdfc2ac6e35c784e844a2bd1c1c152b52feb362c6d877b443ee3f network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-b6878fd46-8ftsx 9a476b33091cdfc2ac6e35c784e844a2bd1c1c152b52feb362c6d877b443ee3f network default NAD default NAD key ] I0425 19:03:35.464291 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-b6878fd46-8ftsx, hostIfaceName: 9a476b33091cdfc, network: default, NAD default, SandboxID: "9a476b33091cdfc2ac6e35c784e844a2bd1c1c152b52feb362c6d877b443ee3f", PCI device ID: , UID: "fa7e0cbb-86ea-4ffd-857c-c334b00e7808", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0425 19:03:35.539861 3656 cni.go:473] [openshift-console/console-b6878fd46-8ftsx 9a476b33091cdfc2ac6e35c784e844a2bd1c1c152b52feb362c6d877b443ee3f network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-b6878fd46-8ftsx 9a476b33091cdfc2ac6e35c784e844a2bd1c1c152b52feb362c6d877b443ee3f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:b4:50:fb:77:7d\",\"name\":\"9a476b33091cdfc\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85a6b31a-b30f-4a88-9f88-8837a145370f\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:03:36.364006 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0425 19:03:46.355488 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0425 19:03:52.549695 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5c56ffbfbd-gbppw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:03:52.550314 3656 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5c56ffbfbd-gbppw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:03:53.205090 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5c56ffbfbd-gbppw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:03:53.205121 3656 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5c56ffbfbd-gbppw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:03:53.212974 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5c56ffbfbd-gbppw I0425 19:03:53.212999 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5c56ffbfbd-gbppw W0425 19:03:56.244931 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0425 19:03:57.252082 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0425 19:04:02.250371 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0425 19:04:03.459881 3656 obj_retry.go:498] Detected object openshift-console/console-6bdc46c845-lpzv8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:03.460513 3656 obj_retry.go:498] Detected object openshift-console/console-6bdc46c845-lpzv8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:04.276471 3656 obj_retry.go:492] Detected object openshift-console/console-6bdc46c845-lpzv8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:04.276499 3656 obj_retry.go:492] Detected object openshift-console/console-6bdc46c845-lpzv8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:04.285465 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6bdc46c845-lpzv8 I0425 19:04:04.285493 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6bdc46c845-lpzv8 I0425 19:04:04.576843 3656 cni.go:452] [openshift-multus/network-metrics-daemon-jszsp f136fc685971e3a4f743e6f335725fabb96ffb9d1db7984f286cfc4a8b012ffe network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-jszsp f136fc685971e3a4f743e6f335725fabb96ffb9d1db7984f286cfc4a8b012ffe network default NAD default NAD key ] I0425 19:04:04.586319 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-jszsp, hostIfaceName: f136fc685971e3a, network: default, NAD default, SandboxID: "f136fc685971e3a4f743e6f335725fabb96ffb9d1db7984f286cfc4a8b012ffe", PCI device ID: , UID: "629cbb39-3b5e-4f1b-a2f0-8a71ba63a649", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0425 19:04:04.635240 3656 cni.go:473] [openshift-multus/network-metrics-daemon-jszsp f136fc685971e3a4f743e6f335725fabb96ffb9d1db7984f286cfc4a8b012ffe network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-jszsp f136fc685971e3a4f743e6f335725fabb96ffb9d1db7984f286cfc4a8b012ffe network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:32:43:7c:bd:53\",\"name\":\"f136fc685971e3a\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb473866-7555-471b-abf3-feda0948d0de\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:04:11.692336 3656 obj_retry.go:498] Detected object openshift-console/console-6bdc46c845-667hj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:11.692869 3656 obj_retry.go:498] Detected object openshift-console/console-6bdc46c845-667hj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:12.310478 3656 obj_retry.go:492] Detected object openshift-console/console-6bdc46c845-667hj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:12.310502 3656 obj_retry.go:492] Detected object openshift-console/console-6bdc46c845-667hj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:12.320702 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6bdc46c845-667hj I0425 19:04:12.320739 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6bdc46c845-667hj I0425 19:04:21.395027 3656 obj_retry.go:498] Detected object openshift-ingress/router-default-547f6f8bcf-5p7w6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:21.395689 3656 obj_retry.go:498] Detected object openshift-ingress/router-default-547f6f8bcf-5p7w6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:22.338296 3656 obj_retry.go:492] Detected object openshift-ingress/router-default-547f6f8bcf-5p7w6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:22.338320 3656 obj_retry.go:492] Detected object openshift-ingress/router-default-547f6f8bcf-5p7w6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:22.347455 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-547f6f8bcf-5p7w6 I0425 19:04:22.347476 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-547f6f8bcf-5p7w6 W0425 19:04:24.388752 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:04:24.388772 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:04:24.388782 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:04:24.388789 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:04:24.532240 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:04:24.532260 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0425 19:04:32.831170 3656 cni.go:452] [openshift-dns/dns-default-rl6sj d9151cf065d2664451ca71aa50fda6c7a38e899b36a23a0d183a48feb54daa0d network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-rl6sj d9151cf065d2664451ca71aa50fda6c7a38e899b36a23a0d183a48feb54daa0d network default NAD default NAD key ] I0425 19:04:32.840557 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-rl6sj, hostIfaceName: d9151cf065d2664, network: default, NAD default, SandboxID: "d9151cf065d2664451ca71aa50fda6c7a38e899b36a23a0d183a48feb54daa0d", PCI device ID: , UID: "59388e49-8fb3-4f48-b350-9135cd49b934", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0425 19:04:32.892976 3656 cni.go:473] [openshift-dns/dns-default-rl6sj d9151cf065d2664451ca71aa50fda6c7a38e899b36a23a0d183a48feb54daa0d network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-rl6sj d9151cf065d2664451ca71aa50fda6c7a38e899b36a23a0d183a48feb54daa0d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:88:8c:a9:e2:83\",\"name\":\"d9151cf065d2664\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/670beb95-7fbc-4a42-ad72-2525faf720a0\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:04:39.054818 3656 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:39.055451 3656 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:39.408334 3656 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:39.408361 3656 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:39.417819 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0425 19:04:39.417844 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0425 19:04:43.376541 3656 cni.go:452] [openshift-monitoring/prometheus-k8s-1 ae84c0fa9c65465c8c37a0fd08a6d24ead97f447b41f902b6cffcec396ff9059 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 ae84c0fa9c65465c8c37a0fd08a6d24ead97f447b41f902b6cffcec396ff9059 network default NAD default NAD key ] I0425 19:04:43.467958 3656 cni.go:473] [openshift-monitoring/prometheus-k8s-1 ae84c0fa9c65465c8c37a0fd08a6d24ead97f447b41f902b6cffcec396ff9059 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 ae84c0fa9c65465c8c37a0fd08a6d24ead97f447b41f902b6cffcec396ff9059 network default NAD default NAD key default], result "{}", err I0425 19:04:43.533088 3656 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:43.533225 3656 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0425 19:04:43.534667 3656 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.128.0.20 I0425 19:04:43.534700 3656 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:04:43.876105 3656 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:43.876129 3656 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:04:43.893529 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0425 19:04:43.893553 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0425 19:04:43.972999 3656 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:04:43.973132 3656 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0425 19:04:43.999326 3656 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 26.232637ms I0425 19:04:43.999354 3656 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 26.264027ms I0425 19:04:44.000650 3656 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 27.672986ms, libovsdb time 924.848µs I0425 19:04:44.338180 3656 cni.go:452] [openshift-monitoring/prometheus-k8s-1 d3f6f1aaae5117e8a60fd3b66e37bf095491f24137f1119cc2d115cf041cfe30 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 d3f6f1aaae5117e8a60fd3b66e37bf095491f24137f1119cc2d115cf041cfe30 network default NAD default NAD key ] I0425 19:04:44.347608 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: d3f6f1aaae5117e, network: default, NAD default, SandboxID: "d3f6f1aaae5117e8a60fd3b66e37bf095491f24137f1119cc2d115cf041cfe30", PCI device ID: , UID: "de067fe9-3cfa-4cee-9400-6f121cd539ba", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0425 19:04:44.399245 3656 cni.go:473] [openshift-monitoring/prometheus-k8s-1 d3f6f1aaae5117e8a60fd3b66e37bf095491f24137f1119cc2d115cf041cfe30 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 d3f6f1aaae5117e8a60fd3b66e37bf095491f24137f1119cc2d115cf041cfe30 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:2f:d6:4e:a2:5f\",\"name\":\"d3f6f1aaae5117e\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ef26315a-1c6d-4d75-aabe-a32e4fbc135b\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:04:45.551905 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0425 19:04:45.551927 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0425 19:04:45.551934 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0425 19:05:11.046406 3656 obj_retry.go:498] Detected object openshift-console/console-b6878fd46-nlptr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:05:11.046947 3656 obj_retry.go:498] Detected object openshift-console/console-b6878fd46-nlptr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:05:11.241644 3656 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:05:11.242194 3656 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:05:11.535371 3656 obj_retry.go:492] Detected object openshift-console/console-b6878fd46-nlptr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:05:11.535394 3656 obj_retry.go:492] Detected object openshift-console/console-b6878fd46-nlptr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:05:11.538664 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-b6878fd46-nlptr I0425 19:05:11.538687 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-b6878fd46-nlptr I0425 19:05:11.564805 3656 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:05:11.564821 3656 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:05:11.579877 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0425 19:05:11.579898 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0425 19:05:16.871628 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=5 I0425 19:05:21.710471 3656 cni.go:452] [openshift-console/console-b6878fd46-8ftsx 9a476b33091cdfc2ac6e35c784e844a2bd1c1c152b52feb362c6d877b443ee3f network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-b6878fd46-8ftsx 9a476b33091cdfc2ac6e35c784e844a2bd1c1c152b52feb362c6d877b443ee3f network default NAD default NAD key ] I0425 19:05:21.794527 3656 cni.go:473] [openshift-console/console-b6878fd46-8ftsx 9a476b33091cdfc2ac6e35c784e844a2bd1c1c152b52feb362c6d877b443ee3f network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-b6878fd46-8ftsx 9a476b33091cdfc2ac6e35c784e844a2bd1c1c152b52feb362c6d877b443ee3f network default NAD default NAD key default], result "{}", err I0425 19:05:21.844787 3656 obj_retry.go:498] Detected object openshift-console/console-b6878fd46-8ftsx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:05:21.844822 3656 pods.go:176] Deleting pod: openshift-console/console-b6878fd46-8ftsx I0425 19:05:21.846174 3656 pods.go:236] Attempting to release IPs for pod: openshift-console/console-b6878fd46-8ftsx, ips: 10.128.0.21 I0425 19:05:21.846201 3656 obj_retry.go:498] Detected object openshift-console/console-b6878fd46-8ftsx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:05:22.743593 3656 obj_retry.go:492] Detected object openshift-console/console-b6878fd46-8ftsx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:05:22.743618 3656 obj_retry.go:492] Detected object openshift-console/console-b6878fd46-8ftsx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:05:22.752663 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-b6878fd46-8ftsx I0425 19:05:22.752705 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-b6878fd46-8ftsx I0425 19:05:26.745543 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=69 I0425 19:05:33.751149 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 W0425 19:05:44.856005 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0425 19:05:44.856026 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0425 19:05:46.167805 3656 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:05:46.168459 3656 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:05:46.678271 3656 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:05:46.678391 3656 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:05:46.688147 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0425 19:05:46.688181 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0425 19:06:07.911271 3656 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 d18e2c6bcebcede17e0cbaa7d7c81e6f828809e52d39dd7e04c9b188a0ca3e9c network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 d18e2c6bcebcede17e0cbaa7d7c81e6f828809e52d39dd7e04c9b188a0ca3e9c network default NAD default NAD key ] I0425 19:06:07.995186 3656 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 d18e2c6bcebcede17e0cbaa7d7c81e6f828809e52d39dd7e04c9b188a0ca3e9c network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 d18e2c6bcebcede17e0cbaa7d7c81e6f828809e52d39dd7e04c9b188a0ca3e9c network default NAD default NAD key default], result "{}", err I0425 19:06:08.040927 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:06:08.040970 3656 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 I0425 19:06:08.042503 3656 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9, ips: 10.128.0.8 I0425 19:06:08.042531 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:06:09.064916 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:06:09.064940 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619060-6l2v9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0425 19:06:37.906154 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0425 19:06:48.951600 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0425 19:06:48.951673 3656 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-t67rv IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-t67rv: no pod IPs found I0425 19:06:53.361212 3656 base_network_controller_pods.go:487] [default/openshift-console/console-74b869859f-4cqwr] creating logical port openshift-console_console-74b869859f-4cqwr for pod on switch ip-10-0-1-130.ec2.internal I0425 19:06:53.361339 3656 kube.go:256] Updating pod openshift-console/console-74b869859f-4cqwr I0425 19:06:53.394377 3656 pod.go:62] [openshift-console/console-74b869859f-4cqwr] pod update took 33.073023ms I0425 19:06:53.394407 3656 base_network_controller_pods.go:951] [default/openshift-console/console-74b869859f-4cqwr] addLogicalPort annotation time took 33.108547ms I0425 19:06:53.395803 3656 pods.go:271] [openshift-console/console-74b869859f-4cqwr] addLogicalPort took 34.610681ms, libovsdb time 973.172µs I0425 19:06:53.720082 3656 cni.go:452] [openshift-console/console-74b869859f-4cqwr cb035a068edb76e1e35a19a0c6fffe30b1d39c3e1c4eb5a3d49ab7be4f74ca97 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-74b869859f-4cqwr cb035a068edb76e1e35a19a0c6fffe30b1d39c3e1c4eb5a3d49ab7be4f74ca97 network default NAD default NAD key ] I0425 19:06:53.729699 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-74b869859f-4cqwr, hostIfaceName: cb035a068edb76e, network: default, NAD default, SandboxID: "cb035a068edb76e1e35a19a0c6fffe30b1d39c3e1c4eb5a3d49ab7be4f74ca97", PCI device ID: , UID: "09371c23-22fd-482a-b1bb-f7c439234acf", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0425 19:06:53.780124 3656 cni.go:473] [openshift-console/console-74b869859f-4cqwr cb035a068edb76e1e35a19a0c6fffe30b1d39c3e1c4eb5a3d49ab7be4f74ca97 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-74b869859f-4cqwr cb035a068edb76e1e35a19a0c6fffe30b1d39c3e1c4eb5a3d49ab7be4f74ca97 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:83:c9:3e:45:6e\",\"name\":\"cb035a068edb76e\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7b2a9a52-c66d-4056-bf8f-e47022cca27b\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:07:04.139532 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0425 19:07:09.834444 3656 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0425 19:07:18.667915 3656 obj_retry.go:498] Detected object openshift-console/console-85f9696b7-4gd5b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:07:18.668529 3656 obj_retry.go:498] Detected object openshift-console/console-85f9696b7-4gd5b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:07:19.055392 3656 obj_retry.go:492] Detected object openshift-console/console-85f9696b7-4gd5b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:07:19.055439 3656 obj_retry.go:492] Detected object openshift-console/console-85f9696b7-4gd5b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:07:19.065901 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-85f9696b7-4gd5b I0425 19:07:19.065933 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-85f9696b7-4gd5b I0425 19:07:29.473595 3656 obj_retry.go:498] Detected object openshift-console/console-85f9696b7-lrnwk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:07:29.474227 3656 obj_retry.go:498] Detected object openshift-console/console-85f9696b7-lrnwk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:07:30.094637 3656 obj_retry.go:492] Detected object openshift-console/console-85f9696b7-lrnwk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:07:30.094665 3656 obj_retry.go:492] Detected object openshift-console/console-85f9696b7-lrnwk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:07:30.103272 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-85f9696b7-lrnwk I0425 19:07:30.103299 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-85f9696b7-lrnwk I0425 19:07:37.866324 3656 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0425 19:07:54.849925 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0425 19:07:58.871227 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0425 19:08:01.744678 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0425 19:08:17.855139 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0425 19:08:18.742614 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=148 I0425 19:08:25.747559 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=80 I0425 19:08:31.844363 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0425 19:08:40.860151 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0425 19:08:49.833660 3656 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0425 19:09:48.838601 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0425 19:10:00.198707 3656 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29619070-98v59 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:10:00.198853 3656 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 I0425 19:10:00.223248 3656 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59] pod update took 24.42788ms I0425 19:10:00.223273 3656 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59] addLogicalPort annotation time took 24.456903ms I0425 19:10:00.224540 3656 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59] addLogicalPort took 25.852715ms, libovsdb time 886.608µs I0425 19:10:00.567073 3656 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 2715df503eb22188327f662e9cf0c6c8abc57ab167c7f70d09d212bc55b1e270 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 2715df503eb22188327f662e9cf0c6c8abc57ab167c7f70d09d212bc55b1e270 network default NAD default NAD key ] I0425 19:10:00.576641 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29619070-98v59, hostIfaceName: 2715df503eb2218, network: default, NAD default, SandboxID: "2715df503eb22188327f662e9cf0c6c8abc57ab167c7f70d09d212bc55b1e270", PCI device ID: , UID: "32e40518-b562-487a-b339-d45ab30fa043", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0425 19:10:00.625279 3656 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 2715df503eb22188327f662e9cf0c6c8abc57ab167c7f70d09d212bc55b1e270 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 2715df503eb22188327f662e9cf0c6c8abc57ab167c7f70d09d212bc55b1e270 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:1f:b2:aa:1e:c5\",\"name\":\"2715df503eb2218\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55ffd38d-3e4f-45be-b470-dc585eb7560a\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:10:03.820412 3656 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 2715df503eb22188327f662e9cf0c6c8abc57ab167c7f70d09d212bc55b1e270 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 2715df503eb22188327f662e9cf0c6c8abc57ab167c7f70d09d212bc55b1e270 network default NAD default NAD key ] I0425 19:10:03.894920 3656 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 2715df503eb22188327f662e9cf0c6c8abc57ab167c7f70d09d212bc55b1e270 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 2715df503eb22188327f662e9cf0c6c8abc57ab167c7f70d09d212bc55b1e270 network default NAD default NAD key default], result "{}", err I0425 19:10:03.941189 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:10:03.941225 3656 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 I0425 19:10:03.942598 3656 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59, ips: 10.128.0.24 I0425 19:10:03.942629 3656 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:10:04.840704 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:10:04.840736 3656 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29619070-98v59 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:10:10.755867 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=668 I0425 19:11:00.922124 3656 namespace.go:98] [tsf] adding namespace I0425 19:11:00.922982 3656 namespace.go:102] [tsf] adding namespace took 833.155µs I0425 19:11:00.948372 3656 namespace.go:142] [tsf] updating namespace I0425 19:11:00.958628 3656 namespace.go:142] [tsf] updating namespace I0425 19:11:04.991424 3656 namespace.go:98] [cert-manager-operator] adding namespace I0425 19:11:04.991553 3656 namespace.go:98] [rhtpa-operator] adding namespace I0425 19:11:04.992412 3656 namespace.go:102] [cert-manager-operator] adding namespace took 894.931µs I0425 19:11:04.992963 3656 namespace.go:102] [rhtpa-operator] adding namespace took 1.390806ms I0425 19:11:04.998323 3656 namespace.go:98] [konflux-operator] adding namespace I0425 19:11:04.998359 3656 namespace.go:98] [konflux-ui] adding namespace I0425 19:11:04.998450 3656 namespace.go:98] [tsf-keycloak] adding namespace I0425 19:11:04.998581 3656 namespace.go:98] [tsf-tas] adding namespace I0425 19:11:04.998683 3656 namespace.go:98] [tsf-tpa] adding namespace I0425 19:11:04.998744 3656 namespace.go:98] [rhbk-operator] adding namespace I0425 19:11:04.999070 3656 namespace.go:102] [konflux-operator] adding namespace took 732.652µs I0425 19:11:04.999638 3656 namespace.go:102] [konflux-ui] adding namespace took 1.267344ms I0425 19:11:05.000158 3656 namespace.go:102] [tsf-keycloak] adding namespace took 1.69054ms I0425 19:11:05.000650 3656 namespace.go:102] [tsf-tas] adding namespace took 2.054742ms I0425 19:11:05.001138 3656 namespace.go:102] [tsf-tpa] adding namespace took 2.441214ms I0425 19:11:05.001823 3656 namespace.go:102] [rhbk-operator] adding namespace took 3.067564ms I0425 19:11:05.012974 3656 namespace.go:142] [cert-manager-operator] updating namespace I0425 19:11:05.027538 3656 namespace.go:142] [cert-manager-operator] updating namespace I0425 19:11:05.034984 3656 namespace.go:142] [rhtpa-operator] updating namespace I0425 19:11:05.050526 3656 namespace.go:142] [rhtpa-operator] updating namespace I0425 19:11:05.058072 3656 namespace.go:142] [konflux-operator] updating namespace I0425 19:11:05.058132 3656 namespace.go:142] [konflux-ui] updating namespace I0425 19:11:05.059363 3656 namespace.go:142] [tsf-keycloak] updating namespace I0425 19:11:05.076152 3656 namespace.go:142] [tsf-tas] updating namespace I0425 19:11:05.079240 3656 namespace.go:142] [rhbk-operator] updating namespace I0425 19:11:05.079609 3656 namespace.go:142] [tsf-tpa] updating namespace I0425 19:11:05.082311 3656 namespace.go:142] [rhtpa-operator] updating namespace I0425 19:11:05.082343 3656 namespace.go:142] [cert-manager-operator] updating namespace I0425 19:11:05.129644 3656 namespace.go:142] [tsf-keycloak] updating namespace I0425 19:11:05.151313 3656 namespace.go:142] [tsf-tas] updating namespace I0425 19:11:05.158568 3656 namespace.go:142] [tsf-keycloak] updating namespace I0425 19:11:05.172966 3656 namespace.go:142] [tsf-tas] updating namespace I0425 19:11:05.177018 3656 namespace.go:142] [tsf-tpa] updating namespace I0425 19:11:05.204256 3656 namespace.go:142] [rhbk-operator] updating namespace I0425 19:11:05.204297 3656 namespace.go:142] [tsf-tpa] updating namespace I0425 19:11:05.230268 3656 namespace.go:142] [konflux-operator] updating namespace I0425 19:11:05.230340 3656 namespace.go:142] [rhbk-operator] updating namespace I0425 19:11:05.242523 3656 namespace.go:142] [konflux-operator] updating namespace I0425 19:11:05.409575 3656 namespace.go:142] [konflux-ui] updating namespace I0425 19:11:05.418059 3656 namespace.go:142] [konflux-ui] updating namespace I0425 19:11:06.382882 3656 namespace.go:142] [tsf-keycloak] updating namespace I0425 19:11:06.399508 3656 namespace.go:142] [tsf-tpa] updating namespace I0425 19:11:15.178774 3656 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2xd8vn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:15.179396 3656 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2xd8vn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:15.179427 3656 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2xd8vn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:15.179436 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 19:11:15.179451 3656 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2xd8vn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:15.179455 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 19:11:16.224165 3656 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2xd8vn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:16.224192 3656 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2xd8vn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:16.224208 3656 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2xd8vn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:16.224222 3656 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2xd8vn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:18.074477 3656 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dssg8g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:18.075090 3656 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dssg8g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:18.075121 3656 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dssg8g of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:18.075132 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 19:11:18.075152 3656 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dssg8g of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:18.075159 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 19:11:19.104722 3656 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dssg8g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:19.104749 3656 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dssg8g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:19.104763 3656 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dssg8g of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:19.104778 3656 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dssg8g of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:20.390286 3656 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158ph59r of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:20.390825 3656 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158ph59r of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:20.390851 3656 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158ph59r of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:20.390859 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 19:11:20.390873 3656 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158ph59r of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:20.390880 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 19:11:21.184994 3656 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c21365hmzk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.185681 3656 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c21365hmzk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.185709 3656 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c21365hmzk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.185719 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 19:11:21.185733 3656 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c21365hmzk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.185741 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 19:11:21.231549 3656 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d778vkb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.232117 3656 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d778vkb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.232165 3656 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d778vkb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.232177 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 19:11:21.232188 3656 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d778vkb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.232193 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 19:11:21.259578 3656 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tvtk6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.260157 3656 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tvtk6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.260192 3656 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tvtk6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.260202 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0425 19:11:21.260217 3656 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tvtk6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0425 19:11:21.260226 3656 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0425 19:11:21.313721 3656 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tvtk6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:21.313742 3656 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tvtk6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:21.313755 3656 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tvtk6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:21.313771 3656 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51tvtk6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:21.417253 3656 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158ph59r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:21.417283 3656 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158ph59r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:21.417294 3656 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158ph59r of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:21.417308 3656 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158ph59r of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:22.214406 3656 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c21365hmzk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:22.214430 3656 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c21365hmzk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:22.214441 3656 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c21365hmzk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:22.214460 3656 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c21365hmzk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:22.265715 3656 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d778vkb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:22.265736 3656 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d778vkb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:22.265750 3656 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d778vkb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:22.265760 3656 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d778vkb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:11:33.290960 3656 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-fxz5w] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-fxz5w for pod on switch ip-10-0-1-130.ec2.internal I0425 19:11:33.291134 3656 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-fxz5w I0425 19:11:33.348261 3656 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-fxz5w] pod update took 57.152432ms I0425 19:11:33.348287 3656 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-fxz5w] addLogicalPort annotation time took 57.185377ms I0425 19:11:33.349491 3656 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-fxz5w] addLogicalPort took 58.552408ms, libovsdb time 885.974µs I0425 19:11:33.955841 3656 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-fxz5w 667e9be1b4a0a51e47d4bda42322f2c39268f831b553efe9ca8c13fa34d51e2e network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-fxz5w 667e9be1b4a0a51e47d4bda42322f2c39268f831b553efe9ca8c13fa34d51e2e network default NAD default NAD key ] I0425 19:11:33.965434 3656 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-fxz5w, hostIfaceName: 667e9be1b4a0a51, network: default, NAD default, SandboxID: "667e9be1b4a0a51e47d4bda42322f2c39268f831b553efe9ca8c13fa34d51e2e", PCI device ID: , UID: "b99e6792-8699-4055-ae56-4a41f78a1b91", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0425 19:11:34.016059 3656 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-fxz5w 667e9be1b4a0a51e47d4bda42322f2c39268f831b553efe9ca8c13fa34d51e2e network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-fxz5w 667e9be1b4a0a51e47d4bda42322f2c39268f831b553efe9ca8c13fa34d51e2e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:c3:97:47:9e:ac\",\"name\":\"667e9be1b4a0a51\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2d633bb5-ce1b-42a1-9698-2b2883f037f4\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:11:37.717690 3656 namespace.go:98] [cert-manager] adding namespace I0425 19:11:37.718589 3656 namespace.go:102] [cert-manager] adding namespace took 873.308µs I0425 19:11:37.845695 3656 namespace.go:142] [cert-manager] updating namespace I0425 19:11:37.881182 3656 namespace.go:142] [cert-manager] updating namespace I0425 19:11:38.793786 3656 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-k47gl] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-k47gl for pod on switch ip-10-0-1-130.ec2.internal I0425 19:11:38.793929 3656 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-k47gl I0425 19:11:38.828150 3656 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-k47gl] pod update took 34.243794ms I0425 19:11:38.828177 3656 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-k47gl] addLogicalPort annotation time took 34.275404ms I0425 19:11:38.829419 3656 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-k47gl] addLogicalPort took 35.653934ms, libovsdb time 864.253µs I0425 19:11:39.169167 3656 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-k47gl c413e21e45edffcf37d8a5c6f1c318af7ea457b3439eba267117aa755558be5e network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-k47gl c413e21e45edffcf37d8a5c6f1c318af7ea457b3439eba267117aa755558be5e network default NAD default NAD key ] I0425 19:11:39.178425 3656 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-k47gl, hostIfaceName: c413e21e45edffc, network: default, NAD default, SandboxID: "c413e21e45edffcf37d8a5c6f1c318af7ea457b3439eba267117aa755558be5e", PCI device ID: , UID: "e7e357fe-6a8a-47de-961f-c6bebbea9d67", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0425 19:11:39.229819 3656 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-k47gl c413e21e45edffcf37d8a5c6f1c318af7ea457b3439eba267117aa755558be5e network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-k47gl c413e21e45edffcf37d8a5c6f1c318af7ea457b3439eba267117aa755558be5e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:9a:a4:b6:ed:16\",\"name\":\"c413e21e45edffc\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a805af4a-76aa-4369-9285-dd4cf60784ea\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:11:43.271494 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0425 19:11:43.271516 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0425 19:11:43.399616 3656 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5d99449ffd-4zmdt] creating logical port cert-manager_cert-manager-cainjector-5d99449ffd-4zmdt for pod on switch ip-10-0-1-130.ec2.internal I0425 19:11:43.399703 3656 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5d99449ffd-4zmdt I0425 19:11:43.419884 3656 pod.go:62] [cert-manager/cert-manager-cainjector-5d99449ffd-4zmdt] pod update took 20.199929ms I0425 19:11:43.419913 3656 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-cainjector-5d99449ffd-4zmdt] addLogicalPort annotation time took 20.235711ms I0425 19:11:43.421179 3656 pods.go:271] [cert-manager/cert-manager-cainjector-5d99449ffd-4zmdt] addLogicalPort took 21.584149ms, libovsdb time 886.482µs I0425 19:11:43.761898 3656 cni.go:452] [cert-manager/cert-manager-cainjector-5d99449ffd-4zmdt 582c4b64fc9c39c92c63ca3d4385f2025490630177625280112b9aac9cdc2699 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-4zmdt 582c4b64fc9c39c92c63ca3d4385f2025490630177625280112b9aac9cdc2699 network default NAD default NAD key ] I0425 19:11:43.771328 3656 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5d99449ffd-4zmdt, hostIfaceName: 582c4b64fc9c39c, network: default, NAD default, SandboxID: "582c4b64fc9c39c92c63ca3d4385f2025490630177625280112b9aac9cdc2699", PCI device ID: , UID: "cca3a434-8fca-441c-be9e-a4f8e49feb84", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0425 19:11:43.823722 3656 cni.go:473] [cert-manager/cert-manager-cainjector-5d99449ffd-4zmdt 582c4b64fc9c39c92c63ca3d4385f2025490630177625280112b9aac9cdc2699 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-4zmdt 582c4b64fc9c39c92c63ca3d4385f2025490630177625280112b9aac9cdc2699 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"22:aa:da:31:9f:c1\",\"name\":\"582c4b64fc9c39c\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/04547042-f571-46ba-854c-f6215af135f0\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:11:45.103796 3656 namespace.go:142] [openshift-operators] updating namespace I0425 19:11:45.120565 3656 namespace.go:142] [openshift-operators] updating namespace W0425 19:11:46.153787 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0425 19:11:49.295032 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0425 19:11:49.295068 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0425 19:11:50.893690 3656 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59c77684cc-tpn2d] creating logical port cert-manager_cert-manager-59c77684cc-tpn2d for pod on switch ip-10-0-1-130.ec2.internal I0425 19:11:50.893786 3656 kube.go:256] Updating pod cert-manager/cert-manager-59c77684cc-tpn2d I0425 19:11:50.959422 3656 pod.go:62] [cert-manager/cert-manager-59c77684cc-tpn2d] pod update took 65.655111ms I0425 19:11:50.959448 3656 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-59c77684cc-tpn2d] addLogicalPort annotation time took 65.691098ms I0425 19:11:50.960907 3656 pods.go:271] [cert-manager/cert-manager-59c77684cc-tpn2d] addLogicalPort took 67.234278ms, libovsdb time 1.001529ms I0425 19:11:51.253358 3656 cni.go:452] [cert-manager/cert-manager-59c77684cc-tpn2d 176e20561699a8ac87aeee52e636b3353b758a21204da97a628e2551fef70dc1 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-59c77684cc-tpn2d 176e20561699a8ac87aeee52e636b3353b758a21204da97a628e2551fef70dc1 network default NAD default NAD key ] I0425 19:11:51.262609 3656 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59c77684cc-tpn2d, hostIfaceName: 176e20561699a8a, network: default, NAD default, SandboxID: "176e20561699a8ac87aeee52e636b3353b758a21204da97a628e2551fef70dc1", PCI device ID: , UID: "46558119-4be8-47b9-b91b-71a73f0e161d", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0425 19:11:51.311025 3656 cni.go:473] [cert-manager/cert-manager-59c77684cc-tpn2d 176e20561699a8ac87aeee52e636b3353b758a21204da97a628e2551fef70dc1 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-59c77684cc-tpn2d 176e20561699a8ac87aeee52e636b3353b758a21204da97a628e2551fef70dc1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:6f:21:06:2c:04\",\"name\":\"176e20561699a8a\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/51a5a74a-03d7-4fc4-8f09-1cbc2dd33569\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:11:51.422940 3656 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-qlbhj] creating logical port openshift-operators_rhtas-operator-controller-manager-68888cb75-qlbhj for pod on switch ip-10-0-1-130.ec2.internal I0425 19:11:51.423077 3656 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-68888cb75-qlbhj I0425 19:11:51.460008 3656 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-68888cb75-qlbhj] pod update took 36.980735ms I0425 19:11:51.460033 3656 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-qlbhj] addLogicalPort annotation time took 37.009981ms I0425 19:11:51.461257 3656 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-68888cb75-qlbhj] addLogicalPort took 38.334609ms, libovsdb time 852.071µs I0425 19:11:51.779873 3656 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-68888cb75-qlbhj 388057bf582dd4855a646767fa24f2644e489301750cc5fa741f782c49ad2a2a network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-qlbhj 388057bf582dd4855a646767fa24f2644e489301750cc5fa741f782c49ad2a2a network default NAD default NAD key ] I0425 19:11:51.789845 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-68888cb75-qlbhj, hostIfaceName: 388057bf582dd48, network: default, NAD default, SandboxID: "388057bf582dd4855a646767fa24f2644e489301750cc5fa741f782c49ad2a2a", PCI device ID: , UID: "e95e847d-2888-4813-965d-efd19a9c7e72", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0425 19:11:51.839476 3656 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-68888cb75-qlbhj 388057bf582dd4855a646767fa24f2644e489301750cc5fa741f782c49ad2a2a network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-qlbhj 388057bf582dd4855a646767fa24f2644e489301750cc5fa741f782c49ad2a2a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:5f:d7:fd:0b:a5\",\"name\":\"388057bf582dd48\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8cf24fa-87fc-47e7-9f85-38ad37e21f12\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:11:52.193434 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0425 19:11:53.468578 3656 namespace.go:98] [trusted-artifact-signer] adding namespace I0425 19:11:53.469486 3656 namespace.go:102] [trusted-artifact-signer] adding namespace took 881.304µs I0425 19:11:53.499219 3656 namespace.go:142] [trusted-artifact-signer] updating namespace I0425 19:11:53.527997 3656 namespace.go:142] [trusted-artifact-signer] updating namespace W0425 19:11:54.300725 3656 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0425 19:11:58.690851 3656 namespace.go:142] [openshift-operators] updating namespace I0425 19:12:00.257775 3656 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-bkvlv] creating logical port openshift-operators_openshift-pipelines-operator-6bdf6c8794-bkvlv for pod on switch ip-10-0-1-130.ec2.internal I0425 19:12:00.257891 3656 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-bkvlv I0425 19:12:00.286877 3656 pod.go:62] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-bkvlv] pod update took 29.012518ms I0425 19:12:00.286902 3656 base_network_controller_pods.go:951] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-bkvlv] addLogicalPort annotation time took 29.042268ms I0425 19:12:00.288144 3656 pods.go:271] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-bkvlv] addLogicalPort took 30.38464ms, libovsdb time 835.593µs I0425 19:12:00.364087 3656 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-bb4t5] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-bb4t5 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:12:00.364199 3656 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-bb4t5 I0425 19:12:00.385463 3656 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-bb4t5] pod update took 21.286807ms I0425 19:12:00.385490 3656 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-bb4t5] addLogicalPort annotation time took 21.3174ms I0425 19:12:00.386765 3656 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-bb4t5] addLogicalPort took 22.695926ms, libovsdb time 921.466µs I0425 19:12:00.618942 3656 cni.go:452] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-bkvlv ef40b09c2eac36144336b1949252f2d0954e3f5e52ec3780f82bd503e9904ad3 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-bkvlv ef40b09c2eac36144336b1949252f2d0954e3f5e52ec3780f82bd503e9904ad3 network default NAD default NAD key ] I0425 19:12:00.628163 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6bdf6c8794-bkvlv, hostIfaceName: ef40b09c2eac361, network: default, NAD default, SandboxID: "ef40b09c2eac36144336b1949252f2d0954e3f5e52ec3780f82bd503e9904ad3", PCI device ID: , UID: "61d2aa6c-0e35-4368-9811-89bcb91d69ad", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0425 19:12:00.679511 3656 cni.go:473] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-bkvlv ef40b09c2eac36144336b1949252f2d0954e3f5e52ec3780f82bd503e9904ad3 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-bkvlv ef40b09c2eac36144336b1949252f2d0954e3f5e52ec3780f82bd503e9904ad3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:4f:05:15:e0:ba\",\"name\":\"ef40b09c2eac361\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90fca9e0-7958-4cdc-babb-30ab88cd6f24\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:12:00.723574 3656 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-bb4t5 738b2c41948b7cfc553da4446c14b81bdaa39eff04157a87e31c1f19625f7eab network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-bb4t5 738b2c41948b7cfc553da4446c14b81bdaa39eff04157a87e31c1f19625f7eab network default NAD default NAD key ] I0425 19:12:00.734403 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-bb4t5, hostIfaceName: 738b2c41948b7cf, network: default, NAD default, SandboxID: "738b2c41948b7cfc553da4446c14b81bdaa39eff04157a87e31c1f19625f7eab", PCI device ID: , UID: "6dfdf246-2c43-40e4-a2bd-ad5907740096", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0425 19:12:00.785632 3656 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-bb4t5 738b2c41948b7cfc553da4446c14b81bdaa39eff04157a87e31c1f19625f7eab network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-bb4t5 738b2c41948b7cfc553da4446c14b81bdaa39eff04157a87e31c1f19625f7eab network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:47:1f:c8:47:01\",\"name\":\"738b2c41948b7cf\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ec9a405e-3516-4710-be44-00ccbfdd6696\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:12:05.325210 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0425 19:12:15.246989 3656 namespace.go:98] [openshift-pipelines] adding namespace I0425 19:12:15.247815 3656 namespace.go:102] [openshift-pipelines] adding namespace took 804.594µs I0425 19:12:15.269022 3656 namespace.go:142] [openshift-pipelines] updating namespace I0425 19:12:16.050674 3656 namespace.go:142] [cert-manager] updating namespace I0425 19:12:16.248702 3656 namespace.go:142] [cert-manager-operator] updating namespace I0425 19:12:16.651513 3656 namespace.go:142] [konflux-operator] updating namespace I0425 19:12:16.850629 3656 namespace.go:142] [konflux-ui] updating namespace I0425 19:12:17.056383 3656 namespace.go:142] [open-cluster-management-2ptbrbue1ohsm606gsl9i41bgt475dp0] updating namespace I0425 19:12:17.454299 3656 namespace.go:142] [rhbk-operator] updating namespace I0425 19:12:17.651808 3656 namespace.go:142] [rhtpa-operator] updating namespace I0425 19:12:17.854892 3656 namespace.go:142] [trusted-artifact-signer] updating namespace I0425 19:12:18.056358 3656 namespace.go:142] [tsf] updating namespace I0425 19:12:18.258630 3656 namespace.go:142] [tsf-keycloak] updating namespace I0425 19:12:18.460838 3656 namespace.go:142] [tsf-tas] updating namespace I0425 19:12:18.661460 3656 namespace.go:142] [tsf-tpa] updating namespace I0425 19:12:18.796844 3656 namespace.go:142] [cert-manager] updating namespace I0425 19:12:18.807460 3656 namespace.go:142] [cert-manager-operator] updating namespace I0425 19:12:18.816585 3656 namespace.go:142] [default] updating namespace I0425 19:12:18.828474 3656 namespace.go:142] [konflux-operator] updating namespace I0425 19:12:18.840717 3656 namespace.go:142] [konflux-ui] updating namespace I0425 19:12:18.850468 3656 namespace.go:142] [open-cluster-management-2ptbrbue1ohsm606gsl9i41bgt475dp0] updating namespace I0425 19:12:18.860352 3656 namespace.go:142] [openshift] updating namespace I0425 19:12:18.871926 3656 namespace.go:142] [rhbk-operator] updating namespace I0425 19:12:18.884207 3656 namespace.go:142] [rhtpa-operator] updating namespace I0425 19:12:18.895594 3656 namespace.go:142] [trusted-artifact-signer] updating namespace I0425 19:12:18.905486 3656 namespace.go:142] [tsf] updating namespace I0425 19:12:18.917573 3656 namespace.go:142] [tsf-keycloak] updating namespace I0425 19:12:18.928422 3656 namespace.go:142] [tsf-tas] updating namespace I0425 19:12:18.939987 3656 namespace.go:142] [tsf-tpa] updating namespace I0425 19:12:19.043014 3656 namespace.go:142] [cert-manager] updating namespace I0425 19:12:19.147445 3656 namespace.go:142] [cert-manager-operator] updating namespace I0425 19:12:19.242451 3656 namespace.go:142] [default] updating namespace I0425 19:12:19.342902 3656 namespace.go:142] [konflux-operator] updating namespace I0425 19:12:19.450441 3656 namespace.go:142] [konflux-ui] updating namespace I0425 19:12:19.544111 3656 namespace.go:142] [open-cluster-management-2ptbrbue1ohsm606gsl9i41bgt475dp0] updating namespace I0425 19:12:19.641942 3656 namespace.go:142] [openshift] updating namespace I0425 19:12:19.739022 3656 namespace.go:142] [rhbk-operator] updating namespace I0425 19:12:19.840966 3656 namespace.go:142] [rhtpa-operator] updating namespace I0425 19:12:19.939792 3656 namespace.go:142] [trusted-artifact-signer] updating namespace I0425 19:12:20.038934 3656 namespace.go:142] [tsf] updating namespace I0425 19:12:20.141580 3656 namespace.go:142] [tsf-keycloak] updating namespace I0425 19:12:20.241453 3656 namespace.go:142] [tsf-tas] updating namespace I0425 19:12:20.340773 3656 namespace.go:142] [tsf-tpa] updating namespace I0425 19:12:20.640604 3656 namespace.go:142] [openshift-pipelines] updating namespace I0425 19:12:21.562901 3656 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:12:21.563486 3656 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:12:21.594469 3656 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:12:21.594497 3656 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:12:21.618750 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0425 19:12:21.618780 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0425 19:12:29.477500 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-c29vg] creating logical port openshift-pipelines_tekton-events-controller-57d7f78b8d-c29vg for pod on switch ip-10-0-1-130.ec2.internal I0425 19:12:29.477617 3656 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-57d7f78b8d-c29vg I0425 19:12:29.497263 3656 pod.go:62] [openshift-pipelines/tekton-events-controller-57d7f78b8d-c29vg] pod update took 19.682974ms I0425 19:12:29.497287 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-c29vg] addLogicalPort annotation time took 19.71456ms I0425 19:12:29.498542 3656 pods.go:271] [openshift-pipelines/tekton-events-controller-57d7f78b8d-c29vg] addLogicalPort took 21.06365ms, libovsdb time 866.766µs I0425 19:12:29.847147 3656 cni.go:452] [openshift-pipelines/tekton-events-controller-57d7f78b8d-c29vg eca7d42065638b56a56102c21a134ff04f8186d6778f4125e2a33f3cb7f4562b network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-c29vg eca7d42065638b56a56102c21a134ff04f8186d6778f4125e2a33f3cb7f4562b network default NAD default NAD key ] I0425 19:12:29.857510 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-57d7f78b8d-c29vg, hostIfaceName: eca7d42065638b5, network: default, NAD default, SandboxID: "eca7d42065638b56a56102c21a134ff04f8186d6778f4125e2a33f3cb7f4562b", PCI device ID: , UID: "92edcb4b-1c5f-42f6-bc9e-611278d470c3", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0425 19:12:29.916774 3656 cni.go:473] [openshift-pipelines/tekton-events-controller-57d7f78b8d-c29vg eca7d42065638b56a56102c21a134ff04f8186d6778f4125e2a33f3cb7f4562b network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-c29vg eca7d42065638b56a56102c21a134ff04f8186d6778f4125e2a33f3cb7f4562b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:78:3a:e0:da:cc\",\"name\":\"eca7d42065638b5\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/787711b3-8eb9-4722-8a69-8840b7d88264\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:12:32.478099 3656 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0425 19:12:32.478120 3656 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0425 19:12:32.478125 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0425 19:12:34.415361 3656 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0425 19:12:34.415381 3656 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0425 19:12:34.415386 3656 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0425 19:12:34.415390 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0425 19:12:34.484895 3656 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0425 19:12:34.484913 3656 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0425 19:12:34.484918 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0425 19:12:37.470738 3656 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0425 19:12:39.408839 3656 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0425 19:12:43.483608 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0425 19:12:43.483632 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0425 19:12:43.483639 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0425 19:12:52.084186 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-785c845664-fmtgn] creating logical port openshift-pipelines_tekton-triggers-controller-785c845664-fmtgn for pod on switch ip-10-0-1-130.ec2.internal I0425 19:12:52.084284 3656 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-785c845664-fmtgn I0425 19:12:52.104096 3656 pod.go:62] [openshift-pipelines/tekton-triggers-controller-785c845664-fmtgn] pod update took 19.837149ms I0425 19:12:52.104121 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-controller-785c845664-fmtgn] addLogicalPort annotation time took 19.868099ms I0425 19:12:52.105395 3656 pods.go:271] [openshift-pipelines/tekton-triggers-controller-785c845664-fmtgn] addLogicalPort took 21.225503ms, libovsdb time 942.227µs I0425 19:12:52.297406 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-mj99f] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-mj99f for pod on switch ip-10-0-1-130.ec2.internal I0425 19:12:52.297511 3656 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-mj99f I0425 19:12:52.316942 3656 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-mj99f] pod update took 19.453963ms I0425 19:12:52.316965 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-mj99f] addLogicalPort annotation time took 19.48379ms I0425 19:12:52.318266 3656 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-mj99f] addLogicalPort took 20.876927ms, libovsdb time 892.274µs I0425 19:12:52.442425 3656 cni.go:452] [openshift-pipelines/tekton-triggers-controller-785c845664-fmtgn 3225183686fd317a0e8d990314b6dcc5932801ca8552c0955fb60d1047db1d7f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-fmtgn 3225183686fd317a0e8d990314b6dcc5932801ca8552c0955fb60d1047db1d7f network default NAD default NAD key ] I0425 19:12:52.452540 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-785c845664-fmtgn, hostIfaceName: 3225183686fd317, network: default, NAD default, SandboxID: "3225183686fd317a0e8d990314b6dcc5932801ca8552c0955fb60d1047db1d7f", PCI device ID: , UID: "6c5d31df-057e-4fa4-9bb0-f2cc187c90b4", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0425 19:12:52.506871 3656 cni.go:473] [openshift-pipelines/tekton-triggers-controller-785c845664-fmtgn 3225183686fd317a0e8d990314b6dcc5932801ca8552c0955fb60d1047db1d7f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-fmtgn 3225183686fd317a0e8d990314b6dcc5932801ca8552c0955fb60d1047db1d7f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:6e:99:14:ab:ae\",\"name\":\"3225183686fd317\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b2f6a7fb-7ff3-434f-880b-5301e78b175e\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:12:52.670820 3656 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-mj99f aa7200386c8f5558e1d892764ab718fffde639e9d7d6ce0b7ce8cddf248f0fcd network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-mj99f aa7200386c8f5558e1d892764ab718fffde639e9d7d6ce0b7ce8cddf248f0fcd network default NAD default NAD key ] I0425 19:12:52.680517 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-mj99f, hostIfaceName: aa7200386c8f555, network: default, NAD default, SandboxID: "aa7200386c8f5558e1d892764ab718fffde639e9d7d6ce0b7ce8cddf248f0fcd", PCI device ID: , UID: "88d5afdc-afa7-4140-81a4-a2772e661ce1", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0425 19:12:52.732184 3656 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-mj99f aa7200386c8f5558e1d892764ab718fffde639e9d7d6ce0b7ce8cddf248f0fcd network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-mj99f aa7200386c8f5558e1d892764ab718fffde639e9d7d6ce0b7ce8cddf248f0fcd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:29:6a:b5:d6:bd\",\"name\":\"aa7200386c8f555\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79ec310f-dd08-4e57-80ab-3b65fa22372f\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:12:55.612055 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0425 19:13:03.746243 3656 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:13:03.746804 3656 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:13:03.791836 3656 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:13:03.791867 3656 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:13:03.800959 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0425 19:13:03.800983 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0425 19:13:05.753924 3656 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0425 19:13:05.754593 3656 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0425 19:13:05.754610 3656 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0425 19:13:05.756625 3656 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0425 19:13:06.607483 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0425 19:13:12.568969 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0425 19:13:12.568989 3656 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0425 19:13:12.569076 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0425 19:13:12.569091 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0425 19:13:12.636095 3656 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0425 19:13:32.808417 3656 namespace.go:142] [openshift-pipelines] updating namespace I0425 19:13:32.837995 3656 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0425 19:13:32.874682 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0425 19:13:34.366274 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppz6t] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-ppz6t for pod on switch ip-10-0-1-130.ec2.internal I0425 19:13:34.366373 3656 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppz6t I0425 19:13:34.384015 3656 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppz6t] pod update took 17.665909ms I0425 19:13:34.384058 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppz6t] addLogicalPort annotation time took 17.715697ms I0425 19:13:34.385242 3656 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppz6t] addLogicalPort took 18.985632ms, libovsdb time 799.514µs I0425 19:13:34.735384 3656 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppz6t ac289e81cb1403d2e6729f71b7d1e4c1eaf52effb1ca76737dc42646270e1eb6 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppz6t ac289e81cb1403d2e6729f71b7d1e4c1eaf52effb1ca76737dc42646270e1eb6 network default NAD default NAD key ] I0425 19:13:34.745007 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-ppz6t, hostIfaceName: ac289e81cb1403d, network: default, NAD default, SandboxID: "ac289e81cb1403d2e6729f71b7d1e4c1eaf52effb1ca76737dc42646270e1eb6", PCI device ID: , UID: "6725fcd9-8be9-4584-b092-4af5a2df3116", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0425 19:13:34.798480 3656 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppz6t ac289e81cb1403d2e6729f71b7d1e4c1eaf52effb1ca76737dc42646270e1eb6 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-ppz6t ac289e81cb1403d2e6729f71b7d1e4c1eaf52effb1ca76737dc42646270e1eb6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:5b:8b:d4:e5:12\",\"name\":\"ac289e81cb1403d\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/df52d6ab-3394-4da6-9346-84609bae5189\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:13:38.959457 3656 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:13:38.960081 3656 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:13:38.981926 3656 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:13:38.981954 3656 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:13:38.992152 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0425 19:13:38.992178 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam W0425 19:13:40.876333 3656 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0425 19:13:40.876350 3656 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0425 19:13:40.876355 3656 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0425 19:13:41.899032 3656 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-4n7xg-f8kzh] creating logical port tsf-tas_ctlog-createtree-job-4n7xg-f8kzh for pod on switch ip-10-0-1-130.ec2.internal I0425 19:13:41.899162 3656 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-4n7xg-f8kzh I0425 19:13:41.918093 3656 pod.go:62] [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh] pod update took 18.954415ms I0425 19:13:41.918118 3656 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-createtree-job-4n7xg-f8kzh] addLogicalPort annotation time took 18.985226ms I0425 19:13:41.919364 3656 pods.go:271] [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh] addLogicalPort took 20.355669ms, libovsdb time 855.877µs I0425 19:13:42.259880 3656 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-bv7bg-mrzmc] creating logical port tsf-tas_rekor-createtree-job-bv7bg-mrzmc for pod on switch ip-10-0-1-130.ec2.internal I0425 19:13:42.259970 3656 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-bv7bg-mrzmc I0425 19:13:42.261051 3656 cni.go:452] [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh 79a972950970d2d9beb2e5183fa1bbe4ec3b19234b165d353e554c9a61c1c071 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh 79a972950970d2d9beb2e5183fa1bbe4ec3b19234b165d353e554c9a61c1c071 network default NAD default NAD key ] I0425 19:13:42.271621 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-4n7xg-f8kzh, hostIfaceName: 79a972950970d2d, network: default, NAD default, SandboxID: "79a972950970d2d9beb2e5183fa1bbe4ec3b19234b165d353e554c9a61c1c071", PCI device ID: , UID: "1e557022-5c31-4677-870a-7172348e76d1", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0425 19:13:42.279692 3656 pod.go:62] [tsf-tas/rekor-createtree-job-bv7bg-mrzmc] pod update took 19.742533ms I0425 19:13:42.279791 3656 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-bv7bg-mrzmc] addLogicalPort annotation time took 19.845995ms I0425 19:13:42.281197 3656 pods.go:271] [tsf-tas/rekor-createtree-job-bv7bg-mrzmc] addLogicalPort took 21.333565ms, libovsdb time 976.326µs I0425 19:13:42.295685 3656 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-6987b6885c-cljq6] creating logical port tsf-tas_trillian-logsigner-6987b6885c-cljq6 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:13:42.295781 3656 kube.go:256] Updating pod tsf-tas/trillian-logsigner-6987b6885c-cljq6 I0425 19:13:42.317087 3656 pod.go:62] [tsf-tas/trillian-logsigner-6987b6885c-cljq6] pod update took 21.32882ms I0425 19:13:42.317179 3656 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logsigner-6987b6885c-cljq6] addLogicalPort annotation time took 21.424293ms I0425 19:13:42.318663 3656 pods.go:271] [tsf-tas/trillian-logsigner-6987b6885c-cljq6] addLogicalPort took 23.000446ms, libovsdb time 983.252µs I0425 19:13:42.332693 3656 cni.go:473] [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh 79a972950970d2d9beb2e5183fa1bbe4ec3b19234b165d353e554c9a61c1c071 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh 79a972950970d2d9beb2e5183fa1bbe4ec3b19234b165d353e554c9a61c1c071 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:e2:14:6c:15:7c\",\"name\":\"79a972950970d2d\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/acee1e49-a26d-40a9-aab3-2e2d8cdc98a0\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:13:42.625646 3656 cni.go:452] [tsf-tas/rekor-createtree-job-bv7bg-mrzmc 4a9b0d93d00e169ed693ef2b9cab221660a03458b6b6dc32f7c20d0d4054e060 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-bv7bg-mrzmc 4a9b0d93d00e169ed693ef2b9cab221660a03458b6b6dc32f7c20d0d4054e060 network default NAD default NAD key ] I0425 19:13:42.635251 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-bv7bg-mrzmc, hostIfaceName: 4a9b0d93d00e169, network: default, NAD default, SandboxID: "4a9b0d93d00e169ed693ef2b9cab221660a03458b6b6dc32f7c20d0d4054e060", PCI device ID: , UID: "b5416d07-1fc1-4f77-bad3-fd78f3d15e71", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0425 19:13:42.688707 3656 cni.go:473] [tsf-tas/rekor-createtree-job-bv7bg-mrzmc 4a9b0d93d00e169ed693ef2b9cab221660a03458b6b6dc32f7c20d0d4054e060 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-bv7bg-mrzmc 4a9b0d93d00e169ed693ef2b9cab221660a03458b6b6dc32f7c20d0d4054e060 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:84:db:12:2a:43\",\"name\":\"4a9b0d93d00e169\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/aabd1978-9d6a-49ae-9979-b8fd49211ff0\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:13:43.264165 3656 cni.go:452] [tsf-tas/trillian-logsigner-6987b6885c-cljq6 2b7fb6c2e59c599a7af3d3f2348d3d8ae64677ed502b07e72842420391c14c93 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logsigner-6987b6885c-cljq6 2b7fb6c2e59c599a7af3d3f2348d3d8ae64677ed502b07e72842420391c14c93 network default NAD default NAD key ] I0425 19:13:43.274433 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-6987b6885c-cljq6, hostIfaceName: 2b7fb6c2e59c599, network: default, NAD default, SandboxID: "2b7fb6c2e59c599a7af3d3f2348d3d8ae64677ed502b07e72842420391c14c93", PCI device ID: , UID: "b9037d1a-d47e-4b8b-bb1a-8a08eb5dde00", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0425 19:13:43.330170 3656 cni.go:473] [tsf-tas/trillian-logsigner-6987b6885c-cljq6 2b7fb6c2e59c599a7af3d3f2348d3d8ae64677ed502b07e72842420391c14c93 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logsigner-6987b6885c-cljq6 2b7fb6c2e59c599a7af3d3f2348d3d8ae64677ed502b07e72842420391c14c93 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:b5:a1:62:53:e9\",\"name\":\"2b7fb6c2e59c599\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cbd1e899-5e33-4049-bee8-ff578f474a7f\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:13:43.865367 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0425 19:13:43.865398 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0425 19:13:44.999354 3656 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-8d68f7f8d-kr6bt] creating logical port tsf-tas_trillian-db-8d68f7f8d-kr6bt for pod on switch ip-10-0-1-130.ec2.internal I0425 19:13:44.999449 3656 kube.go:256] Updating pod tsf-tas/trillian-db-8d68f7f8d-kr6bt I0425 19:13:45.014350 3656 pod.go:62] [tsf-tas/trillian-db-8d68f7f8d-kr6bt] pod update took 14.924149ms I0425 19:13:45.014384 3656 base_network_controller_pods.go:951] [default/tsf-tas/trillian-db-8d68f7f8d-kr6bt] addLogicalPort annotation time took 14.962099ms I0425 19:13:45.015712 3656 pods.go:271] [tsf-tas/trillian-db-8d68f7f8d-kr6bt] addLogicalPort took 16.380732ms, libovsdb time 875.779µs W0425 19:13:49.871237 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0425 19:13:50.459552 3656 cni.go:452] [tsf-tas/trillian-db-8d68f7f8d-kr6bt 6425eba9184769c3d13443e904eecda9f8bda2ad57bb7e692960a60dd777a31b network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-db-8d68f7f8d-kr6bt 6425eba9184769c3d13443e904eecda9f8bda2ad57bb7e692960a60dd777a31b network default NAD default NAD key ] I0425 19:13:50.469797 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-8d68f7f8d-kr6bt, hostIfaceName: 6425eba9184769c, network: default, NAD default, SandboxID: "6425eba9184769c3d13443e904eecda9f8bda2ad57bb7e692960a60dd777a31b", PCI device ID: , UID: "14adb36b-cebf-46b4-9903-6f269d4f2473", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0425 19:13:50.522352 3656 cni.go:473] [tsf-tas/trillian-db-8d68f7f8d-kr6bt 6425eba9184769c3d13443e904eecda9f8bda2ad57bb7e692960a60dd777a31b network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-db-8d68f7f8d-kr6bt 6425eba9184769c3d13443e904eecda9f8bda2ad57bb7e692960a60dd777a31b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:57:8c:cf:a5:8b\",\"name\":\"6425eba9184769c\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b9e8d2c6-55f6-4839-8ea2-a363218a7e6c\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:13:54.954861 3656 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-wqwmn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:13:54.955476 3656 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-wqwmn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:13:55.846146 3656 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-wqwmn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:13:55.846177 3656 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-wqwmn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0425 19:13:56.021839 3656 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0425 19:13:56.911144 3656 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0425 19:13:59.094061 3656 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-dnvcc-j2sv6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:13:59.094698 3656 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-dnvcc-j2sv6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:00.123835 3656 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-dnvcc-j2sv6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:00.123894 3656 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-dnvcc-j2sv6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0425 19:14:07.961222 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql I0425 19:14:20.708238 3656 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fw6cl] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-fw6cl for pod on switch ip-10-0-1-130.ec2.internal I0425 19:14:20.708330 3656 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fw6cl I0425 19:14:20.727842 3656 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fw6cl] pod update took 19.532532ms I0425 19:14:20.727870 3656 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fw6cl] addLogicalPort annotation time took 19.565734ms I0425 19:14:20.729139 3656 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fw6cl] addLogicalPort took 20.917409ms, libovsdb time 800.144µs I0425 19:14:21.068554 3656 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fw6cl d45d278a1f4270613043ba7a7a917375c4d6d31295c8dc75b6aa9c75ec24257d network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fw6cl d45d278a1f4270613043ba7a7a917375c4d6d31295c8dc75b6aa9c75ec24257d network default NAD default NAD key ] I0425 19:14:21.077968 3656 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-fw6cl, hostIfaceName: d45d278a1f42706, network: default, NAD default, SandboxID: "d45d278a1f4270613043ba7a7a917375c4d6d31295c8dc75b6aa9c75ec24257d", PCI device ID: , UID: "18bd01fe-9fbb-40c0-b99b-6849fb57424e", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0425 19:14:21.134402 3656 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fw6cl d45d278a1f4270613043ba7a7a917375c4d6d31295c8dc75b6aa9c75ec24257d network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-fw6cl d45d278a1f4270613043ba7a7a917375c4d6d31295c8dc75b6aa9c75ec24257d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:3f:ad:16:39:73\",\"name\":\"d45d278a1f42706\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/413837eb-5d78-453a-ba56-83950b1c60b0\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0425 19:14:23.067317 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0425 19:14:24.022888 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner W0425 19:14:24.113500 3656 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0425 19:14:24.113519 3656 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0425 19:14:28.297529 3656 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:28.298157 3656 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:28.320396 3656 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:28.320417 3656 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:28.329827 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0425 19:14:28.329851 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0425 19:14:30.749115 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=55 I0425 19:14:33.751419 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=38 W0425 19:14:34.078983 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0425 19:14:37.332507 3656 obj_retry.go:498] Detected object tsf/patch-tekton-config-z8gt6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:37.333084 3656 obj_retry.go:498] Detected object tsf/patch-tekton-config-z8gt6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:38.362575 3656 obj_retry.go:492] Detected object tsf/patch-tekton-config-z8gt6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:38.362607 3656 obj_retry.go:492] Detected object tsf/patch-tekton-config-z8gt6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:40.157343 3656 cni.go:452] [tsf-tas/rekor-createtree-job-bv7bg-mrzmc 4a9b0d93d00e169ed693ef2b9cab221660a03458b6b6dc32f7c20d0d4054e060 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-bv7bg-mrzmc 4a9b0d93d00e169ed693ef2b9cab221660a03458b6b6dc32f7c20d0d4054e060 network default NAD default NAD key ] I0425 19:14:40.242658 3656 cni.go:473] [tsf-tas/rekor-createtree-job-bv7bg-mrzmc 4a9b0d93d00e169ed693ef2b9cab221660a03458b6b6dc32f7c20d0d4054e060 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-bv7bg-mrzmc 4a9b0d93d00e169ed693ef2b9cab221660a03458b6b6dc32f7c20d0d4054e060 network default NAD default NAD key default], result "{}", err I0425 19:14:40.297094 3656 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-bv7bg-mrzmc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:40.297133 3656 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-bv7bg-mrzmc I0425 19:14:40.298486 3656 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-bv7bg-mrzmc, ips: 10.128.0.37 I0425 19:14:40.298512 3656 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-bv7bg-mrzmc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:41.181997 3656 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-bv7bg-mrzmc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:41.182019 3656 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-bv7bg-mrzmc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:44.853961 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0425 19:14:46.397413 3656 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-6f88567d68-9dfbq] creating logical port tsf-tas_rekor-server-6f88567d68-9dfbq for pod on switch ip-10-0-1-130.ec2.internal I0425 19:14:46.397516 3656 kube.go:256] Updating pod tsf-tas/rekor-server-6f88567d68-9dfbq I0425 19:14:46.412511 3656 pod.go:62] [tsf-tas/rekor-server-6f88567d68-9dfbq] pod update took 15.018916ms I0425 19:14:46.412535 3656 base_network_controller_pods.go:951] [default/tsf-tas/rekor-server-6f88567d68-9dfbq] addLogicalPort annotation time took 15.047769ms I0425 19:14:46.413793 3656 pods.go:271] [tsf-tas/rekor-server-6f88567d68-9dfbq] addLogicalPort took 16.403266ms, libovsdb time 868.655µs W0425 19:14:47.076306 3656 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0425 19:14:47.195401 3656 cni.go:452] [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh 79a972950970d2d9beb2e5183fa1bbe4ec3b19234b165d353e554c9a61c1c071 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh 79a972950970d2d9beb2e5183fa1bbe4ec3b19234b165d353e554c9a61c1c071 network default NAD default NAD key ] I0425 19:14:47.283256 3656 cni.go:473] [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh 79a972950970d2d9beb2e5183fa1bbe4ec3b19234b165d353e554c9a61c1c071 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-4n7xg-f8kzh 79a972950970d2d9beb2e5183fa1bbe4ec3b19234b165d353e554c9a61c1c071 network default NAD default NAD key default], result "{}", err I0425 19:14:47.340966 3656 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-4n7xg-f8kzh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:47.341008 3656 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-4n7xg-f8kzh I0425 19:14:47.342369 3656 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-4n7xg-f8kzh, ips: 10.128.0.36 I0425 19:14:47.342397 3656 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-4n7xg-f8kzh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:47.529612 3656 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-pqxnb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:47.530183 3656 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-pqxnb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:47.837202 3656 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0425 19:14:48.216454 3656 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-4n7xg-f8kzh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:48.216476 3656 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-4n7xg-f8kzh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0425 19:14:48.267232 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0425 19:14:48.552798 3656 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-pqxnb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:48.552824 3656 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-pqxnb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:48.597913 3656 obj_retry.go:492] Detected object tsf/patch-tekton-config-z8gt6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:48.597937 3656 obj_retry.go:492] Detected object tsf/patch-tekton-config-z8gt6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:48.602499 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-z8gt6 I0425 19:14:48.602521 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-z8gt6 I0425 19:14:51.859347 3656 cni.go:452] [tsf-tas/rekor-server-6f88567d68-9dfbq 4ca8cd58a2b5d04710694f52975ebc367046783f98a0f1b7c668882a96f5492f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-6f88567d68-9dfbq 4ca8cd58a2b5d04710694f52975ebc367046783f98a0f1b7c668882a96f5492f network default NAD default NAD key ] I0425 19:14:51.868820 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-6f88567d68-9dfbq, hostIfaceName: 4ca8cd58a2b5d04, network: default, NAD default, SandboxID: "4ca8cd58a2b5d04710694f52975ebc367046783f98a0f1b7c668882a96f5492f", PCI device ID: , UID: "ba326ca3-6e8b-4821-8eaf-1bd8219d1e72", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0425 19:14:51.875563 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0425 19:14:51.922766 3656 cni.go:473] [tsf-tas/rekor-server-6f88567d68-9dfbq 4ca8cd58a2b5d04710694f52975ebc367046783f98a0f1b7c668882a96f5492f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-6f88567d68-9dfbq 4ca8cd58a2b5d04710694f52975ebc367046783f98a0f1b7c668882a96f5492f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:e6:a8:ab:87:9b\",\"name\":\"4ca8cd58a2b5d04\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9bb4999-6b00-49fd-b66c-8cc5e799fd2d\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:14:52.183547 3656 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-7884f6c99f-7q7r7] creating logical port tsf-tas_ctlog-7884f6c99f-7q7r7 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:14:52.183641 3656 kube.go:256] Updating pod tsf-tas/ctlog-7884f6c99f-7q7r7 I0425 19:14:52.200867 3656 pod.go:62] [tsf-tas/ctlog-7884f6c99f-7q7r7] pod update took 17.252593ms I0425 19:14:52.200890 3656 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-7884f6c99f-7q7r7] addLogicalPort annotation time took 17.28013ms I0425 19:14:52.202163 3656 pods.go:271] [tsf-tas/ctlog-7884f6c99f-7q7r7] addLogicalPort took 18.635137ms, libovsdb time 870.121µs I0425 19:14:53.158922 3656 cni.go:452] [tsf-tas/ctlog-7884f6c99f-7q7r7 8c538a87dcf93923b9e4a54f09b95e1428285cba2096716883f40ac4ba61ce4f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-7884f6c99f-7q7r7 8c538a87dcf93923b9e4a54f09b95e1428285cba2096716883f40ac4ba61ce4f network default NAD default NAD key ] I0425 19:14:53.170559 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-7884f6c99f-7q7r7, hostIfaceName: 8c538a87dcf9392, network: default, NAD default, SandboxID: "8c538a87dcf93923b9e4a54f09b95e1428285cba2096716883f40ac4ba61ce4f", PCI device ID: , UID: "c8644c11-a44b-468a-b899-b7e12a5385e8", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0425 19:14:53.226459 3656 cni.go:473] [tsf-tas/ctlog-7884f6c99f-7q7r7 8c538a87dcf93923b9e4a54f09b95e1428285cba2096716883f40ac4ba61ce4f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-7884f6c99f-7q7r7 8c538a87dcf93923b9e4a54f09b95e1428285cba2096716883f40ac4ba61ce4f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:41:20:9c:47:5c\",\"name\":\"8c538a87dcf9392\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/33e5bc11-7a16-4bfa-9da4-d666506b6bbe\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:14:53.447658 3656 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:53.448206 3656 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:53.470722 3656 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:53.470748 3656 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:14:53.480089 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0425 19:14:53.480116 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0425 19:14:54.238624 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server W0425 19:14:55.246082 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0425 19:14:59.496867 3656 obj_retry.go:498] Detected object tsf-tpa/create-db-t5sjv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:14:59.497438 3656 obj_retry.go:498] Detected object tsf-tpa/create-db-t5sjv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:15:00.523882 3656 obj_retry.go:492] Detected object tsf-tpa/create-db-t5sjv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:15:00.523905 3656 obj_retry.go:492] Detected object tsf-tpa/create-db-t5sjv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:15:00.859053 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0425 19:15:03.532572 3656 obj_retry.go:498] Detected object tsf-tpa/migrate-db-wrg9z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:15:03.533144 3656 obj_retry.go:498] Detected object tsf-tpa/migrate-db-wrg9z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:15:03.676618 3656 obj_retry.go:492] Detected object tsf-tpa/migrate-db-wrg9z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:15:03.676645 3656 obj_retry.go:492] Detected object tsf-tpa/migrate-db-wrg9z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0425 19:15:05.252087 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0425 19:15:06.720613 3656 obj_retry.go:498] Detected object tsf-tpa/create-importers-qlhjw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:15:06.721183 3656 obj_retry.go:498] Detected object tsf-tpa/create-importers-qlhjw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:15:06.860439 3656 base_network_controller_pods.go:487] [default/tsf-tpa/server-59f4854d79-7sbs8] creating logical port tsf-tpa_server-59f4854d79-7sbs8 for pod on switch ip-10-0-1-130.ec2.internal I0425 19:15:06.860533 3656 kube.go:256] Updating pod tsf-tpa/server-59f4854d79-7sbs8 I0425 19:15:06.877740 3656 pod.go:62] [tsf-tpa/server-59f4854d79-7sbs8] pod update took 17.229706ms I0425 19:15:06.877764 3656 base_network_controller_pods.go:951] [default/tsf-tpa/server-59f4854d79-7sbs8] addLogicalPort annotation time took 17.259777ms I0425 19:15:06.878933 3656 pods.go:271] [tsf-tpa/server-59f4854d79-7sbs8] addLogicalPort took 18.509704ms, libovsdb time 768.126µs I0425 19:15:06.899126 3656 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-gx4br] creating logical port tsf-tpa_importer-86bcdf477b-gx4br for pod on switch ip-10-0-1-130.ec2.internal I0425 19:15:06.899212 3656 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-gx4br I0425 19:15:06.916907 3656 pod.go:62] [tsf-tpa/importer-86bcdf477b-gx4br] pod update took 17.718486ms I0425 19:15:06.916932 3656 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-gx4br] addLogicalPort annotation time took 17.749274ms I0425 19:15:06.918254 3656 pods.go:271] [tsf-tpa/importer-86bcdf477b-gx4br] addLogicalPort took 19.14348ms, libovsdb time 867.878µs W0425 19:15:07.263000 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0425 19:15:07.745656 3656 obj_retry.go:492] Detected object tsf-tpa/create-importers-qlhjw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:15:07.745681 3656 obj_retry.go:492] Detected object tsf-tpa/create-importers-qlhjw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:15:13.522163 3656 cni.go:452] [tsf-tpa/server-59f4854d79-7sbs8 f3848d005ea863617af19bfbf4d68cfd99e67042452131b4a102676394d006fd network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-59f4854d79-7sbs8 f3848d005ea863617af19bfbf4d68cfd99e67042452131b4a102676394d006fd network default NAD default NAD key ] I0425 19:15:13.532290 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-59f4854d79-7sbs8, hostIfaceName: f3848d005ea8636, network: default, NAD default, SandboxID: "f3848d005ea863617af19bfbf4d68cfd99e67042452131b4a102676394d006fd", PCI device ID: , UID: "31106fe1-d74a-45d7-a8b9-e85c1a223b49", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0425 19:15:13.566481 3656 cni.go:452] [tsf-tpa/importer-86bcdf477b-gx4br ccf407a46941a1267ffb2539b88783fdeb4b00d266363a82ad17d7a88cdfa261 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-gx4br ccf407a46941a1267ffb2539b88783fdeb4b00d266363a82ad17d7a88cdfa261 network default NAD default NAD key ] I0425 19:15:13.577926 3656 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-gx4br, hostIfaceName: ccf407a46941a12, network: default, NAD default, SandboxID: "ccf407a46941a1267ffb2539b88783fdeb4b00d266363a82ad17d7a88cdfa261", PCI device ID: , UID: "f43a8587-8e68-4792-9682-3cc97a105d68", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0425 19:15:13.590168 3656 cni.go:473] [tsf-tpa/server-59f4854d79-7sbs8 f3848d005ea863617af19bfbf4d68cfd99e67042452131b4a102676394d006fd network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-59f4854d79-7sbs8 f3848d005ea863617af19bfbf4d68cfd99e67042452131b4a102676394d006fd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:fd:61:d3:8b:55\",\"name\":\"f3848d005ea8636\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1a6be901-5e3f-4668-97f9-6eae40c0b67a\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:15:13.630398 3656 cni.go:473] [tsf-tpa/importer-86bcdf477b-gx4br ccf407a46941a1267ffb2539b88783fdeb4b00d266363a82ad17d7a88cdfa261 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-gx4br ccf407a46941a1267ffb2539b88783fdeb4b00d266363a82ad17d7a88cdfa261 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:f3:73:6c:ef:4d\",\"name\":\"ccf407a46941a12\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/adcbd1e8-1d09-42ef-b30a-1f17b7338869\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0425 19:15:16.748411 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 W0425 19:15:18.390015 3656 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0425 19:15:19.393635 3656 util.go:959] Failed to build node endpoints for node ip-10-0-1-130.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0425 19:15:25.659528 3656 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-gt9cx-nb9jz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:15:25.660107 3656 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-gt9cx-nb9jz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:15:26.541607 3656 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-gt9cx-nb9jz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:15:26.541632 3656 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-gt9cx-nb9jz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:15:31.754895 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=99 W0425 19:15:33.571210 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0425 19:15:34.759179 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=516 I0425 19:15:37.747072 3656 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:15:37.747700 3656 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:15:37.797263 3656 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:15:37.797289 3656 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:15:37.805770 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0425 19:15:37.805801 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0425 19:15:38.679987 3656 namespace.go:98] [default-tenant] adding namespace I0425 19:15:38.680878 3656 namespace.go:102] [default-tenant] adding namespace took 865.889µs I0425 19:15:38.683628 3656 namespace.go:98] [image-controller] adding namespace I0425 19:15:38.684353 3656 namespace.go:102] [image-controller] adding namespace took 706.044µs I0425 19:15:38.685877 3656 namespace.go:98] [build-service] adding namespace I0425 19:15:38.686480 3656 namespace.go:102] [build-service] adding namespace took 588.227µs I0425 19:15:38.686514 3656 namespace.go:98] [integration-service] adding namespace I0425 19:15:38.687145 3656 namespace.go:102] [integration-service] adding namespace took 622.15µs I0425 19:15:38.732756 3656 namespace.go:142] [default-tenant] updating namespace I0425 19:15:38.756855 3656 namespace.go:142] [default-tenant] updating namespace I0425 19:15:38.766306 3656 namespace.go:142] [image-controller] updating namespace I0425 19:15:38.796212 3656 namespace.go:142] [build-service] updating namespace I0425 19:15:38.801578 3656 namespace.go:142] [image-controller] updating namespace I0425 19:15:38.833675 3656 namespace.go:142] [build-service] updating namespace I0425 19:15:38.841835 3656 namespace.go:142] [integration-service] updating namespace I0425 19:15:38.862963 3656 namespace.go:142] [integration-service] updating namespace I0425 19:15:38.980238 3656 namespace.go:142] [build-service] updating namespace I0425 19:15:38.995854 3656 namespace.go:142] [integration-service] updating namespace I0425 19:15:39.021972 3656 namespace.go:98] [release-service] adding namespace I0425 19:15:39.022875 3656 namespace.go:102] [release-service] adding namespace took 878.143µs I0425 19:15:39.057851 3656 namespace.go:142] [konflux-ui] updating namespace I0425 19:15:39.061184 3656 namespace.go:142] [release-service] updating namespace I0425 19:15:39.076571 3656 namespace.go:142] [release-service] updating namespace I0425 19:15:39.288156 3656 namespace.go:98] [konflux-info] adding namespace I0425 19:15:39.288193 3656 namespace.go:98] [namespace-lister] adding namespace I0425 19:15:39.289022 3656 namespace.go:102] [konflux-info] adding namespace took 846.248µs I0425 19:15:39.289800 3656 namespace.go:102] [namespace-lister] adding namespace took 1.593308ms I0425 19:15:39.367493 3656 namespace.go:142] [konflux-info] updating namespace I0425 19:15:39.408471 3656 namespace.go:142] [konflux-info] updating namespace I0425 19:15:39.418589 3656 namespace.go:142] [namespace-lister] updating namespace I0425 19:15:39.440618 3656 namespace.go:98] [enterprise-contract-service] adding namespace I0425 19:15:39.441524 3656 namespace.go:102] [enterprise-contract-service] adding namespace took 884.57µs I0425 19:15:39.462410 3656 namespace.go:142] [image-controller] updating namespace I0425 19:15:39.464909 3656 namespace.go:142] [namespace-lister] updating namespace I0425 19:15:39.496892 3656 namespace.go:142] [enterprise-contract-service] updating namespace I0425 19:15:39.545585 3656 namespace.go:142] [enterprise-contract-service] updating namespace I0425 19:15:39.700688 3656 namespace.go:142] [default-tenant] updating namespace I0425 19:15:39.827559 3656 namespace.go:98] [segment-bridge] adding namespace I0425 19:15:39.828499 3656 namespace.go:102] [segment-bridge] adding namespace took 917.845µs I0425 19:15:39.956104 3656 namespace.go:98] [konflux-cli] adding namespace I0425 19:15:39.956920 3656 namespace.go:102] [konflux-cli] adding namespace took 795.061µs I0425 19:15:40.026717 3656 namespace.go:142] [segment-bridge] updating namespace I0425 19:15:40.123716 3656 namespace.go:142] [segment-bridge] updating namespace I0425 19:15:40.123863 3656 namespace.go:142] [konflux-cli] updating namespace I0425 19:15:40.151632 3656 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0425 19:15:40.152304 3656 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0425 19:15:40.152330 3656 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0425 19:15:40.154962 3656 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0425 19:15:40.176712 3656 namespace.go:142] [konflux-cli] updating namespace I0425 19:15:40.460028 3656 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0425 19:15:40.460080 3656 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0425 19:15:40.462142 3656 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0425 19:15:41.587409 3656 namespace.go:142] [build-service] updating namespace I0425 19:15:42.167645 3656 namespace.go:142] [default-tenant] updating namespace I0425 19:15:42.596913 3656 namespace.go:142] [enterprise-contract-service] updating namespace I0425 19:15:42.868545 3656 namespace.go:142] [image-controller] updating namespace I0425 19:15:43.147129 3656 namespace.go:142] [integration-service] updating namespace I0425 19:15:43.534310 3656 namespace.go:142] [konflux-info] updating namespace I0425 19:15:43.790200 3656 namespace.go:142] [namespace-lister] updating namespace I0425 19:15:44.001148 3656 namespace.go:142] [release-service] updating namespace I0425 19:15:44.147088 3656 namespace.go:142] [build-service] updating namespace I0425 19:15:44.201287 3656 namespace.go:142] [default-tenant] updating namespace I0425 19:15:44.250261 3656 namespace.go:142] [enterprise-contract-service] updating namespace I0425 19:15:44.277484 3656 namespace.go:142] [image-controller] updating namespace I0425 19:15:44.300260 3656 namespace.go:142] [integration-service] updating namespace I0425 19:15:44.354012 3656 namespace.go:142] [konflux-info] updating namespace I0425 19:15:44.385419 3656 namespace.go:142] [namespace-lister] updating namespace I0425 19:15:44.399437 3656 namespace.go:142] [release-service] updating namespace I0425 19:15:44.466435 3656 namespace.go:142] [build-service] updating namespace I0425 19:15:44.549636 3656 namespace.go:142] [default-tenant] updating namespace I0425 19:15:44.621057 3656 namespace.go:142] [enterprise-contract-service] updating namespace I0425 19:15:44.684723 3656 namespace.go:142] [image-controller] updating namespace I0425 19:15:44.750399 3656 namespace.go:142] [integration-service] updating namespace I0425 19:15:44.827355 3656 namespace.go:142] [konflux-info] updating namespace I0425 19:15:44.946251 3656 namespace.go:142] [namespace-lister] updating namespace I0425 19:15:45.220126 3656 namespace.go:142] [release-service] updating namespace W0425 19:15:45.666673 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0425 19:15:46.371749 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0425 19:15:46.551622 3656 namespace.go:142] [konflux-cli] updating namespace I0425 19:15:46.752548 3656 namespace.go:142] [segment-bridge] updating namespace I0425 19:15:46.878220 3656 namespace.go:142] [konflux-cli] updating namespace I0425 19:15:46.895000 3656 namespace.go:142] [segment-bridge] updating namespace I0425 19:15:47.004987 3656 namespace.go:142] [konflux-cli] updating namespace I0425 19:15:47.064867 3656 namespace.go:142] [segment-bridge] updating namespace W0425 19:15:48.754630 3656 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0425 19:15:48.847742 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 W0425 19:15:51.715685 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0425 19:15:51.715818 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0425 19:15:52.722391 3656 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0425 19:15:55.431674 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0425 19:15:55.433357 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0425 19:15:55.461600 3656 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0425 19:15:55.461762 3656 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0425 19:15:56.436770 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0425 19:15:56.436790 3656 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0425 19:15:56.761933 3656 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0425 19:15:56.761952 3656 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0425 19:16:03.039401 3656 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k7nc8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:16:03.040063 3656 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k7nc8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:16:03.471859 3656 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k7nc8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:16:03.471892 3656 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-k7nc8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:16:03.486516 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-k7nc8 I0425 19:16:03.486541 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-k7nc8 I0425 19:16:05.842725 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0425 19:16:24.870113 3656 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0425 19:17:01.361601 3656 obj_retry.go:498] Detected object konflux-ui/proxy-54c7564696-lrfmv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:17:01.362253 3656 obj_retry.go:498] Detected object konflux-ui/proxy-54c7564696-lrfmv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:17:01.782470 3656 obj_retry.go:492] Detected object konflux-ui/proxy-54c7564696-lrfmv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:17:01.782492 3656 obj_retry.go:492] Detected object konflux-ui/proxy-54c7564696-lrfmv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:17:01.794829 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-54c7564696-lrfmv I0425 19:17:01.794857 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-54c7564696-lrfmv I0425 19:17:08.746911 3656 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=154 I0425 19:17:10.382135 3656 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0425 19:17:10.382761 3656 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0425 19:17:10.410754 3656 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:17:10.410780 3656 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0425 19:17:10.419799 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0425 19:17:10.419832 3656 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0425 19:17:36.068990 3656 namespace.go:98] [default-managed-tenant-9emv9] adding namespace I0425 19:17:36.069925 3656 namespace.go:102] [default-managed-tenant-9emv9] adding namespace took 907.648µs I0425 19:17:36.117557 3656 namespace.go:142] [default-managed-tenant-9emv9] updating namespace I0425 19:17:36.131098 3656 namespace.go:142] [default-managed-tenant-9emv9] updating namespace I0425 19:17:37.122686 3656 namespace.go:142] [default-managed-tenant-9emv9] updating namespace I0425 19:17:37.253382 3656 namespace.go:142] [default-managed-tenant-9emv9] updating namespace I0425 19:17:37.352366 3656 namespace.go:142] [default-managed-tenant-9emv9] updating namespace I0425 19:17:54.864388 3656 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10