+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-97.ec2.internal ++ [[ -n ip-10-0-2-97.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-97.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0505 19:46:13.090382113 - disable conntrack on geneve port' I0505 19:46:13.090382113 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0505 19:46:13.142864116 - starting ovnkube-node + echo 'I0505 19:46:13.142864116 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-97.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-97.ec2.internal --init-node ip-10-0-2-97.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-97.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0505 19:46:13.196973 3666 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0505 19:46:13.197034 3666 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-e15cc150f3.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0505 19:46:13.198458 3666 kube.go:431] Waiting for certificate I0505 19:46:13.198505 3666 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0505 19:46:13.198559 3666 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0505 19:46:13.199292 3666 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0505 19:46:13.233249 3666 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0505 19:46:13.233269 3666 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0505 19:46:13.237106 3666 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0505 19:46:13.237194 3666 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-ks4xq" I0505 19:46:13.249596 3666 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-ks4xq" I0505 19:46:13.249656 3666 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0505 19:46:14.199115 3666 kube.go:438] Certificate found I0505 19:46:14.200005 3666 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0505 19:46:14.200959 3666 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0505 19:46:14.258783 3666 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-06 19:41:13 +0000 UTC" deadline="2026-05-06 12:37:31.479009552 +0000 UTC" I0505 19:46:14.258810 3666 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="16h51m17.220207165s" I0505 19:46:14.280032 3666 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0505 19:46:14.288705 3666 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0505 19:46:14.288776 3666 server.go:101] MetricServer registers OVS metrics I0505 19:46:14.288808 3666 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0505 19:46:14.288866 3666 factory.go:561] Starting watch factory I0505 19:46:14.288943 3666 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.288956 3666 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.288961 3666 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.288969 3666 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.288980 3666 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.288985 3666 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.288992 3666 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.289068 3666 server.go:105] MetricServer registers OVN DB metrics I0505 19:46:14.288972 3666 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.288993 3666 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.289495 3666 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.290671 3666 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.290690 3666 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.294230 3666 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0505 19:46:14.294253 3666 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0505 19:46:14.294488 3666 controller_manager.go:370] Starting the ovnkube controller I0505 19:46:14.294498 3666 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-97.ec2.internal I0505 19:46:14.294562 3666 controller_manager.go:395] NBDB zone sync took: 55.171µs I0505 19:46:14.294572 3666 factory.go:561] Starting watch factory I0505 19:46:14.307490 3666 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0505 19:46:14.307511 3666 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0505 19:46:14.311085 3666 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0505 19:46:14.311116 3666 server.go:109] MetricServer registers OVN Controller metrics I0505 19:46:14.315005 3666 server.go:113] MetricServer registers OVN Northd metrics I0505 19:46:14.317193 3666 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.317434 3666 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.317756 3666 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.317945 3666 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.318245 3666 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.320002 3666 metrics.go:519] OVN Metrics Server starts to run ... I0505 19:46:14.325300 3666 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0505 19:46:14.394302 3666 factory.go:1977] *v1.Namespace informer cache synced successfully I0505 19:46:14.394339 3666 factory.go:1977] *v1.Node informer cache synced successfully I0505 19:46:14.394343 3666 factory.go:1977] *v1.Service informer cache synced successfully I0505 19:46:14.394347 3666 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0505 19:46:14.394350 3666 factory.go:1977] *v1.Pod informer cache synced successfully I0505 19:46:14.394353 3666 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0505 19:46:14.394425 3666 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 19:46:14.394430 3666 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 19:46:14.394445 3666 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 19:46:14.394448 3666 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 19:46:14.398488 3666 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 19:46:14.398491 3666 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0505 19:46:14.400298 3666 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0505 19:46:14.400310 3666 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0505 19:46:14.400376 3666 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.400403 3666 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.400427 3666 factory.go:1977] *v1.Namespace informer cache synced successfully I0505 19:46:14.400437 3666 factory.go:1977] *v1.Node informer cache synced successfully I0505 19:46:14.400442 3666 factory.go:1977] *v1.Service informer cache synced successfully I0505 19:46:14.400445 3666 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0505 19:46:14.400449 3666 factory.go:1977] *v1.Pod informer cache synced successfully I0505 19:46:14.400453 3666 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0505 19:46:14.404048 3666 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.406034 3666 factory.go:1977] *v1.EgressIP informer cache synced successfully I0505 19:46:14.406106 3666 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.406118 3666 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.406185 3666 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0505 19:46:14.406196 3666 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0505 19:46:14.409762 3666 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.411737 3666 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0505 19:46:14.411792 3666 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.411803 3666 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.411890 3666 factory.go:1977] *v1.EgressIP informer cache synced successfully I0505 19:46:14.415570 3666 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.417571 3666 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0505 19:46:14.417583 3666 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0505 19:46:14.417649 3666 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.417665 3666 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.420965 3666 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.422972 3666 factory.go:1977] *v1.EgressService informer cache synced successfully I0505 19:46:14.423030 3666 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.423044 3666 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.423055 3666 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0505 19:46:14.426763 3666 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.428725 3666 factory.go:1977] *v1.EgressService informer cache synced successfully I0505 19:46:14.428736 3666 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0505 19:46:14.428887 3666 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0505 19:46:14.428898 3666 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0505 19:46:14.432551 3666 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0505 19:46:14.434537 3666 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0505 19:46:14.434542 3666 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0505 19:46:14.434613 3666 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.434634 3666 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.434629 3666 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.434673 3666 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.438271 3666 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.438556 3666 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0505 19:46:14.440563 3666 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0505 19:46:14.440641 3666 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0505 19:46:14.440649 3666 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0505 19:46:14.440656 3666 factory.go:680] Watch Factory start up complete, took: 151.791464ms I0505 19:46:14.440718 3666 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0505 19:46:14.446066 3666 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0505 19:46:14.446076 3666 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0505 19:46:14.446083 3666 factory.go:680] Watch Factory start up complete, took: 151.509661ms I0505 19:46:14.446091 3666 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-97.ec2.internal" zone I0505 19:46:14.487777 3666 default_node_network_controller.go:710] Initializing the default node network controller I0505 19:46:14.555744 3666 udn_isolation.go:101] Starting UDN host isolation manager I0505 19:46:14.561586 3666 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0505 19:46:14.604124 3666 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0505 19:46:14.604212 3666 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0505 19:46:14.604231 3666 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0505 19:46:14.635898 3666 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0505 19:46:14.635963 3666 default_node_network_controller.go:809] Node ip-10-0-2-97.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0505 19:46:14.641158 3666 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.97"] k8s.ovn.org/zone-name:ip-10-0-2-97.ec2.internal] on node ip-10-0-2-97.ec2.internal I0505 19:46:14.664587 3666 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0505 19:46:14.670641 3666 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0505 19:46:14.670741 3666 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0505 19:46:14.670863 3666 gateway_init.go:263] Preparing Gateway I0505 19:46:14.670875 3666 gateway_shared_intf.go:1687] Creating new gateway I0505 19:46:14.696627 3666 controller_manager.go:429] Waiting for node in zone sync took: 250.523538ms I0505 19:46:14.707760 3666 controller_manager.go:611] Switching to transit router for layer2 networks I0505 19:46:14.707775 3666 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-97.ec2.internal I0505 19:46:14.725645 3666 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0505 19:46:14.731832 3666 controller_manager.go:290] SCTP support detected in OVN I0505 19:46:14.732641 3666 services_controller.go:69] Creating services controller for network=default I0505 19:46:14.732716 3666 udn_enabled_svc.go:80] Waiting for service informer to sync I0505 19:46:14.732775 3666 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0505 19:46:14.732812 3666 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0505 19:46:14.732825 3666 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0505 19:46:14.732836 3666 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0505 19:46:14.733007 3666 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0505 19:46:14.733054 3666 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0505 19:46:14.733092 3666 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 54.954µs I0505 19:46:14.733110 3666 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 48.309µs I0505 19:46:14.733116 3666 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 62.166µs I0505 19:46:14.733122 3666 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0505 19:46:14.733129 3666 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0505 19:46:14.733138 3666 default_network_controller.go:325] Starting the default network controller I0505 19:46:14.733191 3666 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 5.447µs I0505 19:46:14.733210 3666 udn_enabled_svc.go:95] Performing full resync I0505 19:46:14.733481 3666 udn_enabled_svc.go:99] Waiting for handler to sync I0505 19:46:14.733492 3666 udn_enabled_svc.go:104] Starting worker I0505 19:46:14.741877 3666 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0505 19:46:14.741939 3666 repair.go:33] Syncing exgw routes took 50.042µs I0505 19:46:14.741963 3666 default_network_controller.go:414] Starting all the Watchers... I0505 19:46:14.742099 3666 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0505 19:46:14.742111 3666 namespace.go:98] [openshift-config] adding namespace I0505 19:46:14.742114 3666 namespace.go:98] [openshift-host-network] adding namespace I0505 19:46:14.742126 3666 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0505 19:46:14.742128 3666 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0505 19:46:14.742135 3666 namespace.go:98] [openshift-must-gather-operator] adding namespace I0505 19:46:14.742141 3666 namespace.go:98] [openshift-kube-scheduler] adding namespace I0505 19:46:14.742144 3666 namespace.go:98] [openshift-apiserver-operator] adding namespace I0505 19:46:14.742150 3666 namespace.go:98] [kube-system] adding namespace I0505 19:46:14.742155 3666 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0505 19:46:14.742154 3666 namespace.go:98] [openshift-controller-manager] adding namespace I0505 19:46:14.742177 3666 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0505 19:46:14.742101 3666 namespace.go:98] [openshift-console-operator] adding namespace I0505 19:46:14.742224 3666 namespace.go:98] [openshift-multus] adding namespace I0505 19:46:14.742100 3666 namespace.go:98] [openshift-backplane-tam] adding namespace I0505 19:46:14.742667 3666 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 556.703µs I0505 19:46:14.742683 3666 namespace.go:98] [openshift-network-diagnostics] adding namespace W0505 19:46:14.742705 3666 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-5cxt2: pod openshift-network-diagnostics/network-check-target-5cxt2: no pod IPs found W0505 19:46:14.742717 3666 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-knhmv: pod openshift-network-diagnostics/network-check-target-knhmv: no pod IPs found W0505 19:46:14.742723 3666 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-zrrwx: pod openshift-network-diagnostics/network-check-target-zrrwx: no pod IPs found I0505 19:46:14.743189 3666 namespace.go:102] [openshift-network-diagnostics] adding namespace took 501.937µs I0505 19:46:14.743202 3666 namespace.go:98] [openshift-customer-monitoring] adding namespace I0505 19:46:14.743759 3666 namespace.go:102] [openshift-customer-monitoring] adding namespace took 552.525µs I0505 19:46:14.743774 3666 namespace.go:98] [kube-node-lease] adding namespace I0505 19:46:14.744252 3666 namespace.go:102] [openshift-config] adding namespace took 2.134189ms I0505 19:46:14.744261 3666 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0505 19:46:14.744785 3666 namespace.go:102] [openshift-host-network] adding namespace took 2.662785ms I0505 19:46:14.744795 3666 namespace.go:98] [openshift-network-operator] adding namespace I0505 19:46:14.745237 3666 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 3.107953ms I0505 19:46:14.745245 3666 namespace.go:98] [openshift-etcd] adding namespace I0505 19:46:14.745703 3666 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 3.570758ms I0505 19:46:14.745712 3666 namespace.go:98] [openshift-network-console] adding namespace I0505 19:46:14.746177 3666 namespace.go:102] [openshift-must-gather-operator] adding namespace took 4.037882ms I0505 19:46:14.746186 3666 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0505 19:46:14.746657 3666 namespace.go:102] [openshift-kube-scheduler] adding namespace took 4.508028ms I0505 19:46:14.746680 3666 namespace.go:98] [openshift-config-operator] adding namespace I0505 19:46:14.747131 3666 namespace.go:102] [openshift-apiserver-operator] adding namespace took 4.982817ms I0505 19:46:14.747142 3666 namespace.go:98] [openshift-backplane-mobb] adding namespace I0505 19:46:14.747544 3666 namespace.go:102] [kube-system] adding namespace took 5.38705ms I0505 19:46:14.747565 3666 namespace.go:98] [openshift-backplane-cse] adding namespace I0505 19:46:14.748059 3666 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 5.895288ms I0505 19:46:14.748079 3666 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0505 19:46:14.748542 3666 namespace.go:102] [openshift-controller-manager] adding namespace took 6.375508ms I0505 19:46:14.748557 3666 namespace.go:98] [openshift-backplane-csm] adding namespace I0505 19:46:14.749029 3666 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.841649ms I0505 19:46:14.749045 3666 namespace.go:98] [openshift-logging] adding namespace W0505 19:46:14.749446 3666 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-l5n5q: pod openshift-multus/network-metrics-daemon-l5n5q: no pod IPs found W0505 19:46:14.749458 3666 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-vrhdb: pod openshift-multus/network-metrics-daemon-vrhdb: no pod IPs found W0505 19:46:14.749471 3666 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-pcpg9: pod openshift-multus/network-metrics-daemon-pcpg9: no pod IPs found I0505 19:46:14.749485 3666 namespace.go:102] [openshift-console-operator] adding namespace took 7.256047ms I0505 19:46:14.749491 3666 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0505 19:46:14.750023 3666 namespace.go:102] [openshift-multus] adding namespace took 7.790515ms I0505 19:46:14.750039 3666 namespace.go:98] [openshift-console-user-settings] adding namespace I0505 19:46:14.750462 3666 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.226734ms I0505 19:46:14.750475 3666 namespace.go:98] [openshift-kube-apiserver] adding namespace I0505 19:46:14.750904 3666 namespace.go:102] [kube-node-lease] adding namespace took 7.125158ms I0505 19:46:14.750921 3666 namespace.go:98] [openshift-apiserver] adding namespace I0505 19:46:14.751276 3666 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.010029ms I0505 19:46:14.751290 3666 namespace.go:98] [openshift-infra] adding namespace I0505 19:46:14.751699 3666 namespace.go:102] [openshift-network-operator] adding namespace took 6.899892ms I0505 19:46:14.751711 3666 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0505 19:46:14.752124 3666 namespace.go:102] [openshift-etcd] adding namespace took 6.873267ms I0505 19:46:14.752142 3666 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0505 19:46:14.752539 3666 namespace.go:102] [openshift-network-console] adding namespace took 6.820177ms I0505 19:46:14.752556 3666 namespace.go:98] [openshift] adding namespace I0505 19:46:14.752969 3666 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.777572ms I0505 19:46:14.752988 3666 namespace.go:98] [openshift-monitoring] adding namespace I0505 19:46:14.753395 3666 namespace.go:102] [openshift-config-operator] adding namespace took 6.70675ms I0505 19:46:14.753420 3666 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0505 19:46:14.753801 3666 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.653608ms I0505 19:46:14.753814 3666 namespace.go:98] [openshift-authentication-operator] adding namespace I0505 19:46:14.754204 3666 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.63336ms I0505 19:46:14.754224 3666 namespace.go:98] [openshift-insights] adding namespace I0505 19:46:14.754622 3666 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.534204ms I0505 19:46:14.754634 3666 namespace.go:98] [openshift-ingress] adding namespace I0505 19:46:14.755019 3666 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.457148ms I0505 19:46:14.755031 3666 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0505 19:46:14.755451 3666 namespace.go:102] [openshift-logging] adding namespace took 6.397548ms I0505 19:46:14.755466 3666 namespace.go:98] [openshift-backplane-srep] adding namespace I0505 19:46:14.755941 3666 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.44219ms I0505 19:46:14.755958 3666 namespace.go:98] [openshift-config-managed] adding namespace I0505 19:46:14.756380 3666 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.335996ms I0505 19:46:14.756393 3666 namespace.go:98] [dedicated-admin] adding namespace I0505 19:46:14.756810 3666 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.328081ms I0505 19:46:14.756830 3666 namespace.go:98] [openshift-authentication] adding namespace I0505 19:46:14.757310 3666 namespace.go:102] [openshift-apiserver] adding namespace took 6.383028ms I0505 19:46:14.757750 3666 namespace.go:102] [openshift-infra] adding namespace took 6.455162ms I0505 19:46:14.757763 3666 namespace.go:98] [openshift-package-operator] adding namespace I0505 19:46:14.758180 3666 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.463372ms I0505 19:46:14.758194 3666 namespace.go:98] [openshift-marketplace] adding namespace I0505 19:46:14.759021 3666 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.87221ms I0505 19:46:14.759036 3666 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0505 19:46:14.759397 3666 namespace.go:102] [openshift] adding namespace took 6.834023ms I0505 19:46:14.759411 3666 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0505 19:46:14.759785 3666 namespace.go:102] [openshift-monitoring] adding namespace took 6.792267ms I0505 19:46:14.759797 3666 namespace.go:98] [openshift-cluster-version] adding namespace I0505 19:46:14.760203 3666 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.775985ms I0505 19:46:14.760217 3666 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0505 19:46:14.760622 3666 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.803517ms I0505 19:46:14.760635 3666 namespace.go:98] [openshift-backplane-cee] adding namespace I0505 19:46:14.761071 3666 namespace.go:102] [openshift-insights] adding namespace took 6.840879ms I0505 19:46:14.761088 3666 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0505 19:46:14.761468 3666 namespace.go:102] [openshift-ingress] adding namespace took 6.828248ms I0505 19:46:14.761483 3666 namespace.go:98] [openshift-ingress-operator] adding namespace I0505 19:46:14.761911 3666 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.874872ms I0505 19:46:14.761925 3666 namespace.go:98] [openshift-image-registry] adding namespace I0505 19:46:14.762317 3666 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.846347ms I0505 19:46:14.762333 3666 namespace.go:98] [openshift-service-ca-operator] adding namespace I0505 19:46:14.762705 3666 namespace.go:102] [openshift-config-managed] adding namespace took 6.741051ms I0505 19:46:14.762720 3666 namespace.go:98] [openshift-operators] adding namespace I0505 19:46:14.763101 3666 namespace.go:102] [dedicated-admin] adding namespace took 6.702995ms I0505 19:46:14.763115 3666 namespace.go:98] [openshift-machine-api] adding namespace I0505 19:46:14.763449 3666 namespace.go:102] [openshift-authentication] adding namespace took 6.612721ms I0505 19:46:14.763465 3666 namespace.go:98] [openshift-route-controller-manager] adding namespace I0505 19:46:14.763963 3666 namespace.go:102] [openshift-package-operator] adding namespace took 6.194452ms I0505 19:46:14.763977 3666 namespace.go:98] [openshift-ingress-canary] adding namespace I0505 19:46:14.764373 3666 namespace.go:102] [openshift-marketplace] adding namespace took 6.173046ms I0505 19:46:14.764386 3666 namespace.go:98] [openshift-node] adding namespace I0505 19:46:14.764809 3666 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 5.767745ms I0505 19:46:14.764820 3666 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0505 19:46:14.765334 3666 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 5.916942ms I0505 19:46:14.765352 3666 namespace.go:98] [openshift-dns-operator] adding namespace I0505 19:46:14.765770 3666 namespace.go:102] [openshift-cluster-version] adding namespace took 5.966191ms I0505 19:46:14.765785 3666 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0505 19:46:14.766154 3666 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 5.930537ms I0505 19:46:14.766169 3666 namespace.go:98] [openshift-console] adding namespace I0505 19:46:14.766549 3666 namespace.go:102] [openshift-backplane-cee] adding namespace took 5.90685ms I0505 19:46:14.766566 3666 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0505 19:46:14.767057 3666 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 5.963134ms I0505 19:46:14.767072 3666 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0505 19:46:14.767485 3666 namespace.go:102] [openshift-ingress-operator] adding namespace took 5.995334ms I0505 19:46:14.767498 3666 namespace.go:98] [openshift-operators-redhat] adding namespace I0505 19:46:14.767933 3666 namespace.go:102] [openshift-image-registry] adding namespace took 6.000436ms I0505 19:46:14.767967 3666 namespace.go:98] [openshift-dns] adding namespace I0505 19:46:14.768350 3666 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.010863ms I0505 19:46:14.768364 3666 namespace.go:98] [default] adding namespace I0505 19:46:14.768770 3666 namespace.go:102] [openshift-operators] adding namespace took 6.045156ms I0505 19:46:14.768781 3666 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0505 19:46:14.769186 3666 namespace.go:102] [openshift-machine-api] adding namespace took 6.064834ms I0505 19:46:14.769199 3666 namespace.go:98] [openshift-backplane] adding namespace I0505 19:46:14.769596 3666 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.125291ms I0505 19:46:14.769610 3666 namespace.go:98] [kube-public] adding namespace I0505 19:46:14.770018 3666 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.036986ms I0505 19:46:14.770029 3666 namespace.go:98] [openshift-network-node-identity] adding namespace I0505 19:46:14.770425 3666 namespace.go:102] [openshift-node] adding namespace took 6.033326ms I0505 19:46:14.770435 3666 namespace.go:98] [open-cluster-management-2q3veqdi3e5ncvq7hjcncpt81jc24hvs] adding namespace I0505 19:46:14.770899 3666 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.072814ms I0505 19:46:14.771314 3666 namespace.go:102] [openshift-dns-operator] adding namespace took 5.954805ms I0505 19:46:14.771739 3666 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 5.94751ms I0505 19:46:14.771756 3666 namespace.go:98] [openshift-machine-config-operator] adding namespace I0505 19:46:14.772179 3666 namespace.go:102] [openshift-console] adding namespace took 6.003442ms I0505 19:46:14.772610 3666 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.038263ms I0505 19:46:14.773007 3666 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.929443ms I0505 19:46:14.773393 3666 namespace.go:102] [openshift-operators-redhat] adding namespace took 5.889542ms I0505 19:46:14.773409 3666 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0505 19:46:14.773819 3666 namespace.go:102] [openshift-dns] adding namespace took 5.84399ms I0505 19:46:14.774204 3666 namespace.go:102] [default] adding namespace took 5.833711ms I0505 19:46:14.774624 3666 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 5.837959ms I0505 19:46:14.775011 3666 namespace.go:102] [openshift-backplane] adding namespace took 5.806657ms I0505 19:46:14.775391 3666 namespace.go:102] [kube-public] adding namespace took 5.775908ms I0505 19:46:14.775793 3666 namespace.go:102] [openshift-network-node-identity] adding namespace took 5.758565ms I0505 19:46:14.776194 3666 namespace.go:102] [open-cluster-management-2q3veqdi3e5ncvq7hjcncpt81jc24hvs] adding namespace took 5.752541ms I0505 19:46:14.776548 3666 namespace.go:102] [openshift-machine-config-operator] adding namespace took 4.785646ms I0505 19:46:14.777042 3666 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 3.625973ms W0505 19:46:14.777120 3666 master.go:458] Unable to parse local node ip-10-0-2-97.ec2.internal chassis-id annotation. Chassis may be removed during sync W0505 19:46:14.777128 3666 master.go:471] Unable to parse remote node ip-10-0-0-25.ec2.internal chassis-id annotation. Chassis may be removed during sync W0505 19:46:14.777133 3666 master.go:471] Unable to parse remote node ip-10-0-1-250.ec2.internal chassis-id annotation. Chassis may be removed during sync I0505 19:46:14.777137 3666 master.go:481] Removing stale chassis with ID/Name: 78c8c52f-ebcc-4a65-8e88-51aaf1f115b3, hostname: ip-10-0-2-97.ec2.internal I0505 19:46:14.777624 3666 master.go:548] Adding or Updating local node "ip-10-0-2-97.ec2.internal" for network "default" I0505 19:46:14.777739 3666 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-25.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-25.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-25.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-25.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-25.ec2.internal I0505 19:46:14.777750 3666 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-250.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-250.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-250.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-250.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-250.ec2.internal I0505 19:46:14.780739 3666 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-97.ec2.internal for the network default I0505 19:46:14.781022 3666 zone_ic_handler.go:195] Time taken to create transit switch: 269.154µs I0505 19:46:14.781650 3666 default_network_controller.go:823] Node add failed for ip-10-0-2-97.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-97.ec2.internal, error getting gateway config for node ip-10-0-2-97.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-97.ec2.internal", failed to update chassis to local for local node ip-10-0-2-97.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-97.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-97.ec2.internal] E0505 19:46:14.781701 3666 obj_retry.go:575] Failed to create *v1.Node ip-10-0-2-97.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-97.ec2.internal, error getting gateway config for node ip-10-0-2-97.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-97.ec2.internal", failed to update chassis to local for local node ip-10-0-2-97.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-97.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-97.ec2.internal] I0505 19:46:14.781747 3666 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0505 19:46:14.781772 3666 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0505 19:46:14.781758 3666 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-97.ec2.internal", UID:"1d16e4fc-7e2c-4a34-b6a9-9414dea1d975", APIVersion:"v1", ResourceVersion:"8382", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-97.ec2.internal, error getting gateway config for node ip-10-0-2-97.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-97.ec2.internal", failed to update chassis to local for local node ip-10-0-2-97.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-97.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-97.ec2.internal] I0505 19:46:14.781792 3666 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0505 19:46:14.781801 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-25.ec2.internal in network "default" I0505 19:46:14.781827 3666 node_tracker.go:235] Node ip-10-0-0-25.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-25.ec2.internal" I0505 19:46:14.781840 3666 node_tracker.go:169] Node ip-10-0-0-25.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:14.781852 3666 services_controller.go:574] Full service sync requested for network=default I0505 19:46:14.781857 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-250.ec2.internal in network "default" I0505 19:46:14.781865 3666 node_tracker.go:235] Node ip-10-0-1-250.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-250.ec2.internal" I0505 19:46:14.781871 3666 node_tracker.go:169] Node ip-10-0-1-250.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:14.781875 3666 services_controller.go:574] Full service sync requested for network=default I0505 19:46:14.781879 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-97.ec2.internal in network "default" I0505 19:46:14.781886 3666 node_tracker.go:235] Node ip-10-0-2-97.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-97.ec2.internal" I0505 19:46:14.781891 3666 node_tracker.go:169] Node ip-10-0-2-97.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:14.781895 3666 services_controller.go:574] Full service sync requested for network=default I0505 19:46:14.795454 3666 iptables.go:108] Creating table: filter chain: FORWARD I0505 19:46:14.801390 3666 iptables.go:108] Creating table: filter chain: OUTPUT I0505 19:46:14.807478 3666 gateway_shared_intf.go:1816] Gateway Creation Complete I0505 19:46:14.807492 3666 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-97.ec2.internal","mac-address":"0e:7e:8a:da:74:33","ip-addresses":["10.0.2.97/24"],"ip-address":"10.0.2.97/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:78c8c52f-ebcc-4a65-8e88-51aaf1f115b3 k8s.ovn.org/node-encap-ips:["10.0.2.97"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.97/24"} k8s.ovn.org/zone-name:ip-10-0-2-97.ec2.internal] on node ip-10-0-2-97.ec2.internal I0505 19:46:14.825716 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-97.ec2.internal in network "default" I0505 19:46:14.825732 3666 master.go:548] Adding or Updating local node "ip-10-0-2-97.ec2.internal" for network "default" I0505 19:46:14.825773 3666 node_tracker.go:169] Node ip-10-0-2-97.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:14.825787 3666 services_controller.go:574] Full service sync requested for network=default I0505 19:46:14.829635 3666 default_node_network_controller.go:519] Node connection status = connected W0505 19:46:14.830160 3666 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-zrrwx: pod openshift-network-diagnostics/network-check-target-zrrwx: no pod IPs found W0505 19:46:14.830176 3666 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-l5n5q: pod openshift-multus/network-metrics-daemon-l5n5q: no pod IPs found I0505 19:46:14.831186 3666 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-97.ec2.internal for the network default I0505 19:46:14.831365 3666 zone_ic_handler.go:195] Time taken to create transit switch: 164.851µs I0505 19:46:14.882419 3666 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0505 19:46:14.882440 3666 services_controller.go:208] Setting up event handlers for services for network=default I0505 19:46:14.882548 3666 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0505 19:46:14.882615 3666 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0505 19:46:14.882645 3666 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0505 19:46:14.982843 3666 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0505 19:46:14.982864 3666 repair.go:60] Starting repairing loop for services I0505 19:46:14.982958 3666 repair.go:131] Deleted 0 stale service LBs I0505 19:46:14.982977 3666 repair.go:137] Deleted 0 stale Chassis Template Vars I0505 19:46:14.982991 3666 repair.go:62] Finished repairing loop for services: 130.404µs I0505 19:46:14.983021 3666 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0505 19:46:14.983031 3666 services_controller.go:254] Starting workers for network=default I0505 19:46:14.983576 3666 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-knhmv, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-knhmv: suppressed error logged: pod openshift-network-diagnostics/network-check-target-knhmv: no pod IPs found I0505 19:46:14.983596 3666 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-pcpg9, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-pcpg9: suppressed error logged: pod openshift-multus/network-metrics-daemon-pcpg9: no pod IPs found I0505 19:46:14.983603 3666 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-vrhdb, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-vrhdb: suppressed error logged: pod openshift-multus/network-metrics-daemon-vrhdb: no pod IPs found I0505 19:46:14.983609 3666 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-l5n5q] creating logical port openshift-multus_network-metrics-daemon-l5n5q for pod on switch ip-10-0-2-97.ec2.internal I0505 19:46:14.983596 3666 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-5cxt2, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-5cxt2: suppressed error logged: pod openshift-network-diagnostics/network-check-target-5cxt2: no pod IPs found I0505 19:46:14.983649 3666 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-zrrwx] creating logical port openshift-network-diagnostics_network-check-target-zrrwx for pod on switch ip-10-0-2-97.ec2.internal I0505 19:46:14.983809 3666 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-zrrwx I0505 19:46:14.983887 3666 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-l5n5q I0505 19:46:15.000992 3666 pod.go:62] [openshift-network-diagnostics/network-check-target-zrrwx] pod update took 17.275921ms I0505 19:46:15.001015 3666 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-zrrwx] addLogicalPort annotation time took 17.299777ms I0505 19:46:15.001918 3666 pods.go:271] [openshift-network-diagnostics/network-check-target-zrrwx] addLogicalPort took 18.277297ms, libovsdb time 637.484µs I0505 19:46:15.005585 3666 pod.go:62] [openshift-multus/network-metrics-daemon-l5n5q] pod update took 21.890389ms I0505 19:46:15.005603 3666 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-l5n5q] addLogicalPort annotation time took 21.911879ms I0505 19:46:15.006409 3666 pods.go:271] [openshift-multus/network-metrics-daemon-l5n5q] addLogicalPort took 22.805659ms, libovsdb time 589.119µs I0505 19:46:15.006829 3666 admin_network_policy_controller.go:227] Starting controller default-network-controller I0505 19:46:15.006846 3666 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0505 19:46:15.006858 3666 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0505 19:46:15.006862 3666 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0505 19:46:15.007037 3666 repair.go:29] Repairing admin network policies took 169.786µs I0505 19:46:15.007132 3666 repair.go:92] Repairing baseline admin network policies took 87.681µs I0505 19:46:15.007598 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0505 19:46:15.007618 3666 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0505 19:46:15.009652 3666 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0505 19:46:15.009671 3666 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0505 19:46:15.009681 3666 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0505 19:46:15.010963 3666 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0505 19:46:15.010986 3666 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0505 19:46:15.011001 3666 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0505 19:46:15.012065 3666 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0505 19:46:15.012086 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0505 19:46:15.012094 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0505 19:46:15.013562 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0505 19:46:15.013582 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0505 19:46:15.013591 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0505 19:46:15.015042 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0505 19:46:15.015059 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0505 19:46:15.015438 3666 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0505 19:46:15.015454 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0505 19:46:15.016163 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0505 19:46:15.016182 3666 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0505 19:46:15.016189 3666 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0505 19:46:15.017752 3666 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0505 19:46:15.017773 3666 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0505 19:46:15.017781 3666 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0505 19:46:15.018278 3666 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0505 19:46:15.018294 3666 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0505 19:46:15.018304 3666 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0505 19:46:15.019217 3666 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0505 19:46:15.019241 3666 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0505 19:46:15.019256 3666 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0505 19:46:15.021170 3666 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0505 19:46:15.021190 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0505 19:46:15.021199 3666 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0505 19:46:15.021544 3666 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0505 19:46:15.021563 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0505 19:46:15.021570 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0505 19:46:15.022147 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0505 19:46:15.022168 3666 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0505 19:46:15.022176 3666 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0505 19:46:15.023584 3666 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0505 19:46:15.023605 3666 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0505 19:46:15.023617 3666 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0505 19:46:15.024936 3666 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0505 19:46:15.024955 3666 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0505 19:46:15.024965 3666 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0505 19:46:15.026030 3666 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0505 19:46:15.026050 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0505 19:46:15.026059 3666 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0505 19:46:15.026746 3666 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0505 19:46:15.026765 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0505 19:46:15.026773 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0505 19:46:15.027236 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0505 19:46:15.027253 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0505 19:46:15.027259 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0505 19:46:15.028495 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0505 19:46:15.028512 3666 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0505 19:46:15.028533 3666 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0505 19:46:15.029490 3666 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0505 19:46:15.029508 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0505 19:46:15.029531 3666 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0505 19:46:15.029975 3666 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0505 19:46:15.029991 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0505 19:46:15.029998 3666 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0505 19:46:15.030427 3666 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0505 19:46:15.030444 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0505 19:46:15.030451 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0505 19:46:15.030927 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0505 19:46:15.030944 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0505 19:46:15.030963 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0505 19:46:15.031602 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0505 19:46:15.031617 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0505 19:46:15.031623 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0505 19:46:15.032070 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0505 19:46:15.032087 3666 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0505 19:46:15.032093 3666 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0505 19:46:15.032343 3666 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0505 19:46:15.032359 3666 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0505 19:46:15.032367 3666 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0505 19:46:15.033865 3666 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0505 19:46:15.033885 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0505 19:46:15.033895 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0505 19:46:15.034317 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0505 19:46:15.034334 3666 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0505 19:46:15.034340 3666 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0505 19:46:15.034602 3666 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0505 19:46:15.034618 3666 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0505 19:46:15.034631 3666 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0505 19:46:15.035781 3666 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0505 19:46:15.040602 3666 egressfirewall.go:374] Starting EgressFirewall controller I0505 19:46:15.040648 3666 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0505 19:46:15.040649 3666 controller.go:133] Adding controller egress-firewall-controller event handlers I0505 19:46:15.040678 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0505 19:46:15.040693 3666 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0505 19:46:15.040695 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0505 19:46:15.040704 3666 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0505 19:46:15.040981 3666 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0505 19:46:15.040994 3666 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0505 19:46:15.040999 3666 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0505 19:46:15.041007 3666 egressqos.go:193] Setting up event handlers for EgressQoS I0505 19:46:15.041077 3666 egressfirewall.go:723] Syncing node "ip-10-0-0-25.ec2.internal" for egress firewall I0505 19:46:15.041088 3666 egressfirewall.go:723] Syncing node "ip-10-0-1-250.ec2.internal" for egress firewall I0505 19:46:15.041092 3666 egressfirewall.go:723] Syncing node "ip-10-0-2-97.ec2.internal" for egress firewall I0505 19:46:15.041138 3666 egressqos.go:245] Starting EgressQoS Controller I0505 19:46:15.041152 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0505 19:46:15.041162 3666 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0505 19:46:15.041167 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0505 19:46:15.041172 3666 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0505 19:46:15.041178 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0505 19:46:15.041183 3666 shared_informer.go:356] "Caches are synced" controller="egressqos" I0505 19:46:15.041186 3666 egressqos.go:259] Repairing EgressQoSes I0505 19:46:15.041190 3666 egressqos.go:399] Starting repairing loop for egressqos I0505 19:46:15.041264 3666 egressqos.go:401] Finished repairing loop for egressqos: 73.365µs I0505 19:46:15.041278 3666 egressservice_zone.go:132] Setting up event handlers for Egress Services I0505 19:46:15.041308 3666 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-97.ec2.internal I0505 19:46:15.041378 3666 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-97.ec2.internal : 71.957µs I0505 19:46:15.041412 3666 egressservice_zone.go:210] Starting Egress Services Controller I0505 19:46:15.041421 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0505 19:46:15.041426 3666 shared_informer.go:356] "Caches are synced" controller="egressservices" I0505 19:46:15.041432 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0505 19:46:15.041436 3666 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0505 19:46:15.041443 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0505 19:46:15.041447 3666 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0505 19:46:15.041452 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0505 19:46:15.041456 3666 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0505 19:46:15.041460 3666 egressservice_zone.go:228] Repairing Egress Services I0505 19:46:15.041752 3666 master_controller.go:88] Starting Admin Policy Based Route Controller I0505 19:46:15.041764 3666 external_controller.go:278] Starting Admin Policy Based Route Controller I0505 19:46:15.041780 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-25.ec2.internal I0505 19:46:15.041792 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-25.ec2.internal: 14.485µs I0505 19:46:15.041801 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-250.ec2.internal I0505 19:46:15.041807 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-250.ec2.internal: 6.448µs I0505 19:46:15.041814 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-97.ec2.internal I0505 19:46:15.041818 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-97.ec2.internal: 5.45µs I0505 19:46:15.041956 3666 default_network_controller.go:585] Completing all the Watchers took 299.984182ms I0505 19:46:15.041970 3666 default_network_controller.go:589] Starting unidling controllers I0505 19:46:15.041982 3666 unidle.go:46] Registering OVN SB ControllerEvent handler I0505 19:46:15.041990 3666 unidle.go:63] Populating Initial ContollerEvent events I0505 19:46:15.042000 3666 unidle.go:79] Setting up event handlers for services I0505 19:46:15.042086 3666 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0505 19:46:15.330989 3666 default_node_network_controller.go:519] Node connection status = connected I0505 19:46:15.829899 3666 default_node_network_controller.go:519] Node connection status = connected I0505 19:46:16.330043 3666 default_node_network_controller.go:519] Node connection status = connected I0505 19:46:16.492964 3666 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0505 19:46:16.830742 3666 default_node_network_controller.go:519] Node connection status = connected I0505 19:46:16.847999 3666 bridgeconfig.go:537] Gateway is ready I0505 19:46:16.848017 3666 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0505 19:46:16.859270 3666 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.97. Updating OVN encap IP. I0505 19:46:16.864415 3666 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.97 - it is already configured I0505 19:46:16.864426 3666 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.97/24:{} 172.20.0.1/32:{}]. Updating annotations. I0505 19:46:16.864654 3666 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.97/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-97.ec2.internal","mac-address":"0e:7e:8a:da:74:33","ip-addresses":["10.0.2.97/24"],"ip-address":"10.0.2.97/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:78c8c52f-ebcc-4a65-8e88-51aaf1f115b3 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.97/24"}] on node ip-10-0-2-97.ec2.internal I0505 19:46:16.886105 3666 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.97 I0505 19:46:16.886178 3666 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-97.ec2.internal I0505 19:46:16.886362 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-97.ec2.internal I0505 19:46:16.886381 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-97.ec2.internal: 23.048µs I0505 19:46:16.886501 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-97.ec2.internal in network "default" I0505 19:46:16.886504 3666 egressfirewall.go:723] Syncing node "ip-10-0-2-97.ec2.internal" for egress firewall I0505 19:46:16.886508 3666 master.go:548] Adding or Updating local node "ip-10-0-2-97.ec2.internal" for network "default" I0505 19:46:16.886584 3666 node_tracker.go:169] Node ip-10-0-2-97.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:16.886599 3666 services_controller.go:574] Full service sync requested for network=default I0505 19:46:16.890746 3666 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-97.ec2.internal. Updating no re-route policies I0505 19:46:16.890811 3666 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-97.ec2.internal I0505 19:46:16.890826 3666 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-97.ec2.internal I0505 19:46:16.890835 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-s4nwl I0505 19:46:16.890844 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-l5n5q I0505 19:46:16.890857 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-l5n5q I0505 19:46:16.890856 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-ktsfm I0505 19:46:16.890861 3666 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-wgfnx I0505 19:46:16.890865 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-kd9nn I0505 19:46:16.890869 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-ktsfm I0505 19:46:16.890832 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-6g6gc I0505 19:46:16.890875 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-kd9nn I0505 19:46:16.890877 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-vrhdb I0505 19:46:16.890881 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bn655 I0505 19:46:16.890880 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-m45dh I0505 19:46:16.890848 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-s4nwl I0505 19:46:16.890893 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-kd9nn after 0 failed attempt(s) I0505 19:46:16.890895 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-m45dh I0505 19:46:16.890919 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-s4nwl after 0 failed attempt(s) I0505 19:46:16.890882 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-vrhdb I0505 19:46:16.890938 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-ktsfm after 0 failed attempt(s) I0505 19:46:16.890855 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-zrrwx I0505 19:46:16.890826 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-fgsrp I0505 19:46:16.890959 3666 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-vrhdb, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-vrhdb: suppressed error logged: pod openshift-multus/network-metrics-daemon-vrhdb: no pod IPs found I0505 19:46:16.890921 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-m45dh after 0 failed attempt(s) I0505 19:46:16.890868 3666 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-wgfnx I0505 19:46:16.890969 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-fgsrp I0505 19:46:16.890978 3666 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-wgfnx after 0 failed attempt(s) I0505 19:46:16.890867 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-w747r I0505 19:46:16.890987 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-w747r I0505 19:46:16.890997 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-w747r after 0 failed attempt(s) I0505 19:46:16.891002 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-fgsrp after 0 failed attempt(s) I0505 19:46:16.890869 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-5cxt2 I0505 19:46:16.890812 3666 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-97.ec2.internal I0505 19:46:16.891025 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-97.ec2.internal I0505 19:46:16.891046 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-97.ec2.internal after 0 failed attempt(s) I0505 19:46:16.890889 3666 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-l5n5q] creating logical port openshift-multus_network-metrics-daemon-l5n5q for pod on switch ip-10-0-2-97.ec2.internal I0505 19:46:16.890887 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bn655 I0505 19:46:16.890950 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-zrrwx I0505 19:46:16.891191 3666 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-zrrwx] creating logical port openshift-network-diagnostics_network-check-target-zrrwx for pod on switch ip-10-0-2-97.ec2.internal I0505 19:46:16.890856 3666 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-97.ec2.internal after 0 failed attempt(s) I0505 19:46:16.891016 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-5cxt2 I0505 19:46:16.891313 3666 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-5cxt2, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-5cxt2: suppressed error logged: pod openshift-network-diagnostics/network-check-target-5cxt2: no pod IPs found I0505 19:46:16.890886 3666 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-6g6gc I0505 19:46:16.891333 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-6g6gc after 0 failed attempt(s) I0505 19:46:16.891164 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-bn655 after 0 failed attempt(s) I0505 19:46:16.891710 3666 pods.go:271] [openshift-multus/network-metrics-daemon-l5n5q] addLogicalPort took 831.204µs, libovsdb time 260.854µs I0505 19:46:16.891720 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-l5n5q after 0 failed attempt(s) I0505 19:46:16.891988 3666 pods.go:271] [openshift-network-diagnostics/network-check-target-zrrwx] addLogicalPort took 802.529µs, libovsdb time 480.991µs I0505 19:46:16.891997 3666 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-zrrwx after 0 failed attempt(s) I0505 19:46:16.911550 3666 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0505 19:46:16.922993 3666 iptables.go:108] Creating table: mangle chain: OUTPUT I0505 19:46:16.926060 3666 iptables.go:108] Creating table: nat chain: OUTPUT I0505 19:46:16.979026 3666 iptables.go:108] Creating table: nat chain: PREROUTING I0505 19:46:17.031349 3666 iptables.go:108] Creating table: filter chain: FORWARD I0505 19:46:17.047831 3666 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0505 19:46:17.048068 3666 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0505 19:46:17.048170 3666 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0505 19:46:17.048291 3666 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0505 19:46:17.048407 3666 gateway_init.go:325] Gateway and management port readiness took 2.22277651s I0505 19:46:17.048436 3666 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0505 19:46:17.048470 3666 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0505 19:46:17.048481 3666 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0505 19:46:17.048490 3666 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0505 19:46:17.048565 3666 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0505 19:46:17.048647 3666 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 52.873µs I0505 19:46:17.048888 3666 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0505 19:46:17.048912 3666 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.277µs I0505 19:46:17.048920 3666 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 33.094µs I0505 19:46:17.048926 3666 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0505 19:46:17.048934 3666 nad_controller.go:379] [node-nad-controller NAD controller]: started I0505 19:46:17.048939 3666 default_node_network_controller.go:899] Starting the default node network controller I0505 19:46:17.048944 3666 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0505 19:46:17.049022 3666 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 12.963µs I0505 19:46:17.049109 3666 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0505 19:46:17.049118 3666 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0505 19:46:17.049140 3666 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000d1c180 0xc0002c7a40 0xc0002c7b20 0xc0002c7b90 0xc0002c7c00 0xc0002c7c70 0xc0002c7ce0 0xc00037bd60 0xc0002c7d50 0xc0002c7e30 0xc000aac230 0xc006450cb0 0xc0002c7ea0 map[0x2ff9080:0xc0040ac120 0x2ff95c0:0xc0040ac2a0 0x2ffa2e0:0xc0040ac060 0x2ffa580:0xc0040ac0c0 0x2ffad60:0xc0040ac180 0x2ffb000:0xc0040ac1e0 0x2ffb2a0:0xc0040ac240 0x2ffba80:0xc0025915c0 0x2ffbd20:0xc002591620 0x3026980:0xc000fb7440 0x3026d20:0xc0024109c0 0x3027800:0xc000d4e600 0x3029160:0xc001ba8000 0x30298a0:0xc0040ac000 0x302a720:0xc000d126c0] 0xc0009fc4d0 0} I0505 19:46:17.049194 3666 gateway.go:151] Starting gateway service sync I0505 19:46:17.049662 3666 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0505 19:46:17.051739 3666 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0505 19:46:17.072990 3666 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0505 19:46:17.074115 3666 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0505 19:46:17.075216 3666 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0505 19:46:17.117147 3666 gateway.go:168] Gateway service sync done. Time taken: 67.938519ms I0505 19:46:17.117637 3666 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30996 I0505 19:46:17.117908 3666 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000d1c180 0xc0002c7a40 0xc0002c7b20 0xc0002c7b90 0xc0002c7c00 0xc0002c7c70 0xc0002c7ce0 0xc00037bd60 0xc0002c7d50 0xc0002c7e30 0xc000aac230 0xc006450cb0 0xc0002c7ea0 map[0x2ff9080:0xc0040ac120 0x2ff95c0:0xc0040ac2a0 0x2ffa2e0:0xc0040ac060 0x2ffa580:0xc0040ac0c0 0x2ffad60:0xc0040ac180 0x2ffb000:0xc0040ac1e0 0x2ffb2a0:0xc0040ac240 0x2ffba80:0xc0025915c0 0x2ffbd20:0xc002591620 0x3026980:0xc000fb7440 0x3026d20:0xc0024109c0 0x3027800:0xc000d4e600 0x3029160:0xc001ba8000 0x30298a0:0xc0040ac000 0x302a720:0xc000d126c0] 0xc0009fc4d0 0} I0505 19:46:17.118354 3666 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000d1c180 0xc0002c7a40 0xc0002c7b20 0xc0002c7b90 0xc0002c7c00 0xc0002c7c70 0xc0002c7ce0 0xc00037bd60 0xc0002c7d50 0xc0002c7e30 0xc000aac230 0xc006450cb0 0xc0002c7ea0 map[0x2ff9080:0xc0040ac120 0x2ff95c0:0xc0040ac2a0 0x2ffa2e0:0xc0040ac060 0x2ffa580:0xc0040ac0c0 0x2ffad60:0xc0040ac180 0x2ffb000:0xc0040ac1e0 0x2ffb2a0:0xc0040ac240 0x2ffba80:0xc0025915c0 0x2ffbd20:0xc002591620 0x3026980:0xc000fb7440 0x3026d20:0xc0024109c0 0x3027800:0xc000d4e600 0x3029160:0xc001ba8000 0x30298a0:0xc0040ac000 0x302a720:0xc000d126c0] 0xc0009fc4d0 0} I0505 19:46:17.118393 3666 gateway.go:532] Reconciling gateway with updates I0505 19:46:17.123992 3666 gateway.go:339] Spawning Conntrack Rule Check Thread I0505 19:46:17.124074 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0505 19:46:17.124082 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0505 19:46:17.124087 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0505 19:46:17.124095 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0505 19:46:17.124098 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0505 19:46:17.124102 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0505 19:46:17.124110 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0505 19:46:17.124113 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0505 19:46:17.124115 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0505 19:46:17.124130 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0505 19:46:17.124135 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0505 19:46:17.124140 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0505 19:46:17.124142 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0505 19:46:17.124158 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0505 19:46:17.124165 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0505 19:46:17.124163 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0505 19:46:17.124164 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0505 19:46:17.124172 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0505 19:46:17.124159 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0505 19:46:17.124178 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0505 19:46:17.124181 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0505 19:46:17.124183 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0505 19:46:17.124186 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0505 19:46:17.124188 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0505 19:46:17.124190 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0505 19:46:17.124193 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 19:46:17.124131 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0505 19:46:17.124205 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0505 19:46:17.124206 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 19:46:17.124211 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0505 19:46:17.124211 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0505 19:46:17.124174 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0505 19:46:17.124222 3666 port_claim.go:71] Svc openshift-ingress/router-default: port 31059 is already open, no action needed I0505 19:46:17.124228 3666 port_claim.go:71] Svc openshift-ingress/router-default: port 31074 is already open, no action needed I0505 19:46:17.124228 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0505 19:46:17.124228 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0505 19:46:17.124231 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0505 19:46:17.124231 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0505 19:46:17.124237 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0505 19:46:17.124241 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0505 19:46:17.124245 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0505 19:46:17.124249 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0505 19:46:17.124124 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0505 19:46:17.124197 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0505 19:46:17.124262 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0505 19:46:17.124261 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0505 19:46:17.124273 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0505 19:46:17.124274 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0505 19:46:17.124276 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0505 19:46:17.124280 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0505 19:46:17.124284 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0505 19:46:17.124222 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0505 19:46:17.124292 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0505 19:46:17.124295 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0505 19:46:17.124169 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0505 19:46:17.124299 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0505 19:46:17.124300 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0505 19:46:17.124303 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0505 19:46:17.124309 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0505 19:46:17.124309 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0505 19:46:17.124314 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0505 19:46:17.124318 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0505 19:46:17.124322 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0505 19:46:17.124329 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 19:46:17.124265 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0505 19:46:17.124340 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 19:46:17.124194 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0505 19:46:17.124340 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0505 19:46:17.124357 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0505 19:46:17.124100 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0505 19:46:17.124365 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0505 19:46:17.124299 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0505 19:46:17.124371 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0505 19:46:17.124313 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0505 19:46:17.124199 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0505 19:46:17.124231 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0505 19:46:17.124411 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0505 19:46:17.124419 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0505 19:46:17.124418 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0505 19:46:17.124453 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0505 19:46:17.124458 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0505 19:46:17.124463 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0505 19:46:17.124254 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0505 19:46:17.124475 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0505 19:46:17.124490 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0505 19:46:17.124499 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0505 19:46:17.124254 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0505 19:46:17.124510 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0505 19:46:17.124255 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0505 19:46:17.124533 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0505 19:46:17.124286 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0505 19:46:17.124082 3666 default_node_network_controller.go:944] Gateway and management port readiness took 74.956163ms I0505 19:46:17.124289 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0505 19:46:17.124675 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0505 19:46:17.124175 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0505 19:46:17.124746 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0505 19:46:17.124760 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0505 19:46:17.124301 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0505 19:46:17.124375 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0505 19:46:17.124216 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0505 19:46:17.124829 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0505 19:46:17.124399 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0505 19:46:17.124860 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0505 19:46:17.124402 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0505 19:46:17.124895 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0505 19:46:17.124234 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0505 19:46:17.124251 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0505 19:46:17.124921 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0505 19:46:17.125070 3666 node_ip_handler_linux.go:160] Node IP manager is running W0505 19:46:17.166433 3666 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0505 19:46:17.166839 3666 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0505 19:46:17.166987 3666 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0505 19:46:17.174783 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0505 19:46:17.174793 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0505 19:46:17.174809 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0505 19:46:17.174816 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0505 19:46:17.174821 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0505 19:46:17.174828 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0505 19:46:17.174833 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0505 19:46:17.174839 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0505 19:46:17.174845 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0505 19:46:17.174851 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0505 19:46:17.174856 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0505 19:46:17.174861 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0505 19:46:17.174866 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0505 19:46:17.174872 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0505 19:46:17.214085 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0505 19:46:17.214123 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0505 19:46:17.214136 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0505 19:46:17.214144 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0505 19:46:17.214153 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0505 19:46:17.214161 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0505 19:46:17.214170 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0505 19:46:17.255212 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0505 19:46:17.255238 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0505 19:46:17.255244 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0505 19:46:17.255251 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0505 19:46:17.255257 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0505 19:46:17.255262 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0505 19:46:17.764849 3666 egressfirewall.go:723] Syncing node "ip-10-0-0-25.ec2.internal" for egress firewall I0505 19:46:17.764856 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-25.ec2.internal I0505 19:46:17.764883 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-25.ec2.internal: 32.206µs I0505 19:46:17.764894 3666 default_network_controller.go:1025] Node "ip-10-0-0-25.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0505 19:46:19.084924 3666 egressip_healthcheck.go:76] Health checking using insecure connection I0505 19:46:19.084943 3666 default_node_network_controller.go:1380] Starting node controller node sync I0505 19:46:19.084977 3666 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0505 19:46:19.096778 3666 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.820274ms I0505 19:46:19.096908 3666 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0505 19:46:19.097189 3666 egressservice_node.go:93] Setting up event handlers for Egress Services I0505 19:46:19.097359 3666 egressservice_node.go:183] Starting Egress Services Controller I0505 19:46:19.097376 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0505 19:46:19.097387 3666 shared_informer.go:356] "Caches are synced" controller="egressservices" I0505 19:46:19.097393 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0505 19:46:19.097398 3666 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0505 19:46:19.097404 3666 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0505 19:46:19.097408 3666 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0505 19:46:19.097412 3666 egressservice_node.go:197] Repairing Egress Services I0505 19:46:19.107578 3666 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0505 19:46:19.107588 3666 external_controller.go:278] Starting Admin Policy Based Route Controller I0505 19:46:19.107710 3666 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0505 19:46:19.107721 3666 link_network_manager.go:119] Link manager is running I0505 19:46:19.107731 3666 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0505 19:46:19.107926 3666 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0505 19:46:19.108023 3666 vrf_manager.go:128] VRF manager is running I0505 19:46:19.108106 3666 node_controller_manager.go:437] Removing flows to drop GARP I0505 19:46:19.108115 3666 gateway.go:532] Reconciling gateway with updates I0505 19:46:19.125902 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0505 19:46:19.125915 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0505 19:46:19.125926 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0505 19:46:19.125929 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0505 19:46:19.125934 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0505 19:46:19.125920 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0505 19:46:19.125941 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0505 19:46:19.125945 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0505 19:46:19.125948 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0505 19:46:19.125952 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0505 19:46:19.125956 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0505 19:46:19.125957 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0505 19:46:19.125957 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0505 19:46:19.125969 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0505 19:46:19.125973 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0505 19:46:19.125979 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0505 19:46:19.125981 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0505 19:46:19.125981 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0505 19:46:19.125981 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0505 19:46:19.125923 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0505 19:46:19.125997 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0505 19:46:19.126000 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0505 19:46:19.126000 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0505 19:46:19.126006 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0505 19:46:19.125999 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0505 19:46:19.126009 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0505 19:46:19.126012 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0505 19:46:19.126022 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0505 19:46:19.126024 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0505 19:46:19.125923 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0505 19:46:19.126032 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0505 19:46:19.125970 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0505 19:46:19.126042 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0505 19:46:19.126043 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0505 19:46:19.126045 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0505 19:46:19.126045 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0505 19:46:19.125902 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0505 19:46:19.126001 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0505 19:46:19.126062 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0505 19:46:19.126067 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0505 19:46:19.126067 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0505 19:46:19.126067 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0505 19:46:19.126072 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0505 19:46:19.126076 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0505 19:46:19.126077 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0505 19:46:19.126080 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0505 19:46:19.126080 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0505 19:46:19.126086 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0505 19:46:19.126092 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0505 19:46:19.126095 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0505 19:46:19.126084 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0505 19:46:19.125946 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0505 19:46:19.126110 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0505 19:46:19.126110 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0505 19:46:19.126013 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0505 19:46:19.126120 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0505 19:46:19.126072 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0505 19:46:19.126154 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0505 19:46:19.126174 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 19:46:19.126017 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0505 19:46:19.125958 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0505 19:46:19.126191 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0505 19:46:19.126191 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 19:46:19.126006 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0505 19:46:19.126215 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0505 19:46:19.126022 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0505 19:46:19.126226 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0505 19:46:19.125936 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0505 19:46:19.126266 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0505 19:46:19.125940 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0505 19:46:19.126284 3666 port_claim.go:71] Svc openshift-ingress/router-default: port 31059 is already open, no action needed I0505 19:46:19.126290 3666 port_claim.go:71] Svc openshift-ingress/router-default: port 31074 is already open, no action needed I0505 19:46:19.125910 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0505 19:46:19.126332 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0505 19:46:19.125947 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0505 19:46:19.125957 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0505 19:46:19.126372 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0505 19:46:19.126376 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0505 19:46:19.125989 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0505 19:46:19.126014 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0505 19:46:19.126032 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0505 19:46:19.126410 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0505 19:46:19.126415 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0505 19:46:19.126034 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0505 19:46:19.126033 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0505 19:46:19.126048 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 19:46:19.126455 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 19:46:19.126054 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0505 19:46:19.126055 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0505 19:46:19.126495 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0505 19:46:19.126055 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0505 19:46:19.126503 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0505 19:46:19.126506 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0505 19:46:19.126056 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0505 19:46:19.126063 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0505 19:46:19.126556 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0505 19:46:19.126059 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0505 19:46:19.126069 3666 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0505 19:46:19.126582 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0505 19:46:19.126076 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0505 19:46:19.126074 3666 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0505 19:46:19.126626 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0505 19:46:19.150078 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0505 19:46:19.150101 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0505 19:46:19.150111 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0505 19:46:19.150120 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0505 19:46:19.150127 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0505 19:46:19.150142 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0505 19:46:19.150161 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0505 19:46:19.150170 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0505 19:46:19.150178 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0505 19:46:19.150186 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0505 19:46:19.150196 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0505 19:46:19.150204 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0505 19:46:19.150212 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0505 19:46:19.150221 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0505 19:46:19.150229 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0505 19:46:19.150237 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0505 19:46:19.150244 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0505 19:46:19.150253 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0505 19:46:19.150260 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0505 19:46:19.150268 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0505 19:46:19.191339 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0505 19:46:19.213107 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0505 19:46:19.213153 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0505 19:46:19.213173 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0505 19:46:19.213181 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0505 19:46:19.213189 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0505 19:46:19.213197 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0505 19:46:19.213205 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0505 19:46:19.213214 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0505 19:46:19.213223 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0505 19:46:19.213230 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0505 19:46:19.213237 3666 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0505 19:46:19.560972 3666 default_network_controller.go:1025] Node "ip-10-0-0-25.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 19:46:20.511054 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-97.ec2.internal I0505 19:46:20.511091 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-97.ec2.internal: 44.866µs I0505 19:46:20.543908 3666 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5bb69d5545-xm7sg] creating logical port openshift-image-registry_image-registry-5bb69d5545-xm7sg for pod on switch ip-10-0-2-97.ec2.internal I0505 19:46:20.544074 3666 kube.go:256] Updating pod openshift-image-registry/image-registry-5bb69d5545-xm7sg I0505 19:46:20.567029 3666 pod.go:62] [openshift-image-registry/image-registry-5bb69d5545-xm7sg] pod update took 23.023667ms I0505 19:46:20.567053 3666 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-5bb69d5545-xm7sg] addLogicalPort annotation time took 23.051642ms I0505 19:46:20.568676 3666 pods.go:271] [openshift-image-registry/image-registry-5bb69d5545-xm7sg] addLogicalPort took 24.783271ms, libovsdb time 1.270432ms I0505 19:46:20.598940 3666 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-8sgmm] creating logical port openshift-dns_dns-default-8sgmm for pod on switch ip-10-0-2-97.ec2.internal I0505 19:46:20.598995 3666 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-mqmlp] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-mqmlp for pod on switch ip-10-0-2-97.ec2.internal I0505 19:46:20.599076 3666 kube.go:256] Updating pod openshift-dns/dns-default-8sgmm I0505 19:46:20.599082 3666 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-mqmlp I0505 19:46:20.599091 3666 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-jfz4w] creating logical port openshift-ingress-canary_ingress-canary-jfz4w for pod on switch ip-10-0-2-97.ec2.internal I0505 19:46:20.599190 3666 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-jfz4w I0505 19:46:20.621422 3666 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mqmlp] pod update took 22.368703ms I0505 19:46:20.621445 3666 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-mqmlp] addLogicalPort annotation time took 22.397675ms I0505 19:46:20.622621 3666 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mqmlp] addLogicalPort took 23.639384ms, libovsdb time 847.971µs I0505 19:46:20.625263 3666 pod.go:62] [openshift-dns/dns-default-8sgmm] pod update took 26.2216ms I0505 19:46:20.625288 3666 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-8sgmm] addLogicalPort annotation time took 26.250637ms I0505 19:46:20.625308 3666 pod.go:62] [openshift-ingress-canary/ingress-canary-jfz4w] pod update took 26.145896ms I0505 19:46:20.625322 3666 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-jfz4w] addLogicalPort annotation time took 26.1638ms I0505 19:46:20.626545 3666 pods.go:271] [openshift-ingress-canary/ingress-canary-jfz4w] addLogicalPort took 27.464475ms, libovsdb time 957.56µs I0505 19:46:20.627066 3666 pods.go:271] [openshift-dns/dns-default-8sgmm] addLogicalPort took 28.143708ms, libovsdb time 1.465395ms I0505 19:46:21.035608 3666 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mqmlp 05f1b30bb0978d340bc64f94950f03d6ed9b9e27b141fb46ad4ba27ffdcba4d8 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mqmlp 05f1b30bb0978d340bc64f94950f03d6ed9b9e27b141fb46ad4ba27ffdcba4d8 network default NAD default NAD key ] I0505 19:46:21.036855 3666 cni.go:452] [openshift-ingress-canary/ingress-canary-jfz4w f1b7eb6511145208f73b441c9d76729119fd7901ecd959cd24e35ca27803e656 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-jfz4w f1b7eb6511145208f73b441c9d76729119fd7901ecd959cd24e35ca27803e656 network default NAD default NAD key ] I0505 19:46:21.070775 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-jfz4w, hostIfaceName: f1b7eb651114520, network: default, NAD default, SandboxID: "f1b7eb6511145208f73b441c9d76729119fd7901ecd959cd24e35ca27803e656", PCI device ID: , UID: "30e5e31a-2728-4ba8-a2b5-35790e11f1ff", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0505 19:46:21.070835 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-mqmlp, hostIfaceName: 05f1b30bb0978d3, network: default, NAD default, SandboxID: "05f1b30bb0978d340bc64f94950f03d6ed9b9e27b141fb46ad4ba27ffdcba4d8", PCI device ID: , UID: "501b1d18-8bac-4df5-863c-4088a361fbd5", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0505 19:46:21.121324 3666 cni.go:473] [openshift-ingress-canary/ingress-canary-jfz4w f1b7eb6511145208f73b441c9d76729119fd7901ecd959cd24e35ca27803e656 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-jfz4w f1b7eb6511145208f73b441c9d76729119fd7901ecd959cd24e35ca27803e656 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:b1:29:7f:22:18\",\"name\":\"f1b7eb651114520\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc3e50b6-83ac-40a7-aa8f-15db8c82f1e9\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:46:21.126319 3666 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mqmlp 05f1b30bb0978d340bc64f94950f03d6ed9b9e27b141fb46ad4ba27ffdcba4d8 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-mqmlp 05f1b30bb0978d340bc64f94950f03d6ed9b9e27b141fb46ad4ba27ffdcba4d8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:71:88:5c:51:39\",\"name\":\"05f1b30bb0978d3\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/811c9800-3613-4081-b604-2203292a492e\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:46:22.269846 3666 default_network_controller.go:1025] Node "ip-10-0-0-25.ec2.internal" in remote zone "ip-10-0-0-25.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 19:46:22.269881 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-25.ec2.internal in network "default" I0505 19:46:22.270192 3666 node_tracker.go:235] Node ip-10-0-0-25.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-25.ec2.internal" I0505 19:46:22.270208 3666 node_tracker.go:169] Node ip-10-0-0-25.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:22.270220 3666 services_controller.go:574] Full service sync requested for network=default I0505 19:46:22.301975 3666 default_network_controller.go:1025] Node "ip-10-0-0-25.ec2.internal" in remote zone "ip-10-0-0-25.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 19:46:22.412248 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-25.ec2.internal in network "default" I0505 19:46:22.412264 3666 default_network_controller.go:1025] Node "ip-10-0-0-25.ec2.internal" in remote zone "ip-10-0-0-25.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 19:46:22.412312 3666 node_tracker.go:169] Node ip-10-0-0-25.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:22.412325 3666 services_controller.go:574] Full service sync requested for network=default I0505 19:46:22.413401 3666 zone_ic_handler.go:195] Time taken to create transit switch: 304.843µs I0505 19:46:22.413416 3666 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-25.ec2.internal for the network default I0505 19:46:22.414569 3666 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-25.ec2.internal" on network "default" took: 1.48766ms I0505 19:46:23.177805 3666 cni.go:452] [openshift-network-diagnostics/network-check-target-zrrwx 4b9e868024a13accfa5443377df7b6483f4829cc55b033f54cb03b9b79bcd28f network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-zrrwx 4b9e868024a13accfa5443377df7b6483f4829cc55b033f54cb03b9b79bcd28f network default NAD default NAD key ] I0505 19:46:23.188470 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-zrrwx, hostIfaceName: 4b9e868024a13ac, network: default, NAD default, SandboxID: "4b9e868024a13accfa5443377df7b6483f4829cc55b033f54cb03b9b79bcd28f", PCI device ID: , UID: "482713f7-ed7a-4300-9c8e-26cfdaddee99", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0505 19:46:23.232923 3666 cni.go:473] [openshift-network-diagnostics/network-check-target-zrrwx 4b9e868024a13accfa5443377df7b6483f4829cc55b033f54cb03b9b79bcd28f network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-zrrwx 4b9e868024a13accfa5443377df7b6483f4829cc55b033f54cb03b9b79bcd28f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:db:4c:c1:d6:65\",\"name\":\"4b9e868024a13ac\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ea312ee6-2a2e-4869-be11-c236b264b3d4\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:46:24.304509 3666 default_network_controller.go:1025] Node "ip-10-0-1-250.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 19:46:24.969077 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-25.ec2.internal in network "default" I0505 19:46:24.969142 3666 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-25.ec2.internal", IP: 10.0.0.25 to PMTUD blocking rules I0505 19:46:24.969157 3666 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-25.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0505 19:46:24.969180 3666 node_tracker.go:169] Node ip-10-0-0-25.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:24.969201 3666 services_controller.go:574] Full service sync requested for network=default I0505 19:46:24.969093 3666 egressfirewall.go:723] Syncing node "ip-10-0-0-25.ec2.internal" for egress firewall I0505 19:46:24.969096 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-25.ec2.internal I0505 19:46:24.969235 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-25.ec2.internal: 138.014µs I0505 19:46:24.975964 3666 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-25.ec2.internal. Updating no re-route policies I0505 19:46:24.988710 3666 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-25.ec2.internal", IP: 10.0.0.25 to PMTUD blocking rules I0505 19:46:24.988727 3666 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-25.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0505 19:46:28.408235 3666 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0505 19:46:28.607430 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-25.ec2.internal I0505 19:46:28.607465 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-25.ec2.internal: 41.954µs W0505 19:46:29.407860 3666 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 19:46:32.933547 3666 default_network_controller.go:1025] Node "ip-10-0-1-250.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 19:46:32.933600 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-250.ec2.internal I0505 19:46:32.933612 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-250.ec2.internal: 16.173µs I0505 19:46:32.933600 3666 egressfirewall.go:723] Syncing node "ip-10-0-1-250.ec2.internal" for egress firewall I0505 19:46:34.958294 3666 default_network_controller.go:1025] Node "ip-10-0-1-250.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0505 19:46:37.916361 3666 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0505 19:46:38.349768 3666 default_network_controller.go:1025] Node "ip-10-0-1-250.ec2.internal" in remote zone "ip-10-0-1-250.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 19:46:38.349833 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-250.ec2.internal in network "default" I0505 19:46:38.349853 3666 node_tracker.go:235] Node ip-10-0-1-250.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-250.ec2.internal" I0505 19:46:38.349868 3666 node_tracker.go:169] Node ip-10-0-1-250.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:38.349882 3666 services_controller.go:574] Full service sync requested for network=default W0505 19:46:38.350073 3666 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 19:46:38.350173 3666 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 19:46:38.498447 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-250.ec2.internal in network "default" I0505 19:46:38.498500 3666 node_tracker.go:169] Node ip-10-0-1-250.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:38.498447 3666 default_network_controller.go:1025] Node "ip-10-0-1-250.ec2.internal" in remote zone "ip-10-0-1-250.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0505 19:46:38.498534 3666 services_controller.go:574] Full service sync requested for network=default W0505 19:46:38.498817 3666 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 19:46:38.498866 3666 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 19:46:38.499456 3666 zone_ic_handler.go:195] Time taken to create transit switch: 276.483µs I0505 19:46:38.499470 3666 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-250.ec2.internal for the network default I0505 19:46:38.500452 3666 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-250.ec2.internal" on network "default" took: 1.283112ms I0505 19:46:40.056654 3666 egressfirewall.go:723] Syncing node "ip-10-0-1-250.ec2.internal" for egress firewall I0505 19:46:40.056654 3666 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-250.ec2.internal in network "default" I0505 19:46:40.056708 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-250.ec2.internal I0505 19:46:40.056724 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-250.ec2.internal: 20.754µs I0505 19:46:40.056728 3666 node_tracker.go:169] Node ip-10-0-1-250.ec2.internal switch + router changed, syncing services in network "default" I0505 19:46:40.056732 3666 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-250.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0505 19:46:40.056742 3666 services_controller.go:574] Full service sync requested for network=default I0505 19:46:40.056747 3666 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-250.ec2.internal", IP: 10.0.1.250 to PMTUD blocking rules W0505 19:46:40.057104 3666 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 19:46:40.057161 3666 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 19:46:40.062867 3666 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-250.ec2.internal. Updating no re-route policies I0505 19:46:40.075449 3666 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-250.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0505 19:46:40.075463 3666 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-250.ec2.internal", IP: 10.0.1.250 to PMTUD blocking rules W0505 19:46:41.434278 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0505 19:46:43.743087 3666 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-250.ec2.internal I0505 19:46:43.743115 3666 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-250.ec2.internal: 35.413µs W0505 19:46:53.088815 3666 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0505 19:46:59.413927 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0505 19:48:27.861255 3666 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7c7b844789-db9r8] creating logical port openshift-ingress_router-default-7c7b844789-db9r8 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:48:27.861376 3666 kube.go:256] Updating pod openshift-ingress/router-default-7c7b844789-db9r8 I0505 19:48:27.862164 3666 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7df9d57-9m8mn] creating logical port openshift-ingress_router-default-7df9d57-9m8mn for pod on switch ip-10-0-2-97.ec2.internal I0505 19:48:27.862265 3666 kube.go:256] Updating pod openshift-ingress/router-default-7df9d57-9m8mn I0505 19:48:27.970933 3666 kube.go:256] Updating pod openshift-ingress/router-default-7c7b844789-db9r8 I0505 19:48:27.980089 3666 kube.go:256] Updating pod openshift-ingress/router-default-7df9d57-9m8mn I0505 19:48:28.029033 3666 pod.go:62] [openshift-ingress/router-default-7c7b844789-db9r8] pod update took 167.691089ms I0505 19:48:28.029053 3666 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-7c7b844789-db9r8] addLogicalPort annotation time took 167.714856ms I0505 19:48:28.030214 3666 pods.go:271] [openshift-ingress/router-default-7c7b844789-db9r8] addLogicalPort took 168.975597ms, libovsdb time 873.535µs I0505 19:48:28.080563 3666 pod.go:62] [openshift-ingress/router-default-7df9d57-9m8mn] pod update took 218.323162ms I0505 19:48:28.080583 3666 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-7df9d57-9m8mn] addLogicalPort annotation time took 218.34591ms I0505 19:48:28.081503 3666 pods.go:271] [openshift-ingress/router-default-7df9d57-9m8mn] addLogicalPort took 219.350972ms, libovsdb time 704.805µs I0505 19:48:28.218089 3666 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-dnqkt] creating logical port openshift-insights_insights-operator-6dbb575d6c-dnqkt for pod on switch ip-10-0-2-97.ec2.internal I0505 19:48:28.218132 3666 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-cdnv5 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:48:28.218189 3666 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-dnqkt I0505 19:48:28.218205 3666 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5 I0505 19:48:28.305212 3666 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-dnqkt I0505 19:48:28.312339 3666 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5 I0505 19:48:28.357828 3666 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-dnqkt] pod update took 139.668471ms I0505 19:48:28.357849 3666 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-dnqkt] addLogicalPort annotation time took 139.696211ms I0505 19:48:28.358775 3666 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-dnqkt] addLogicalPort took 140.715068ms, libovsdb time 689.513µs I0505 19:48:28.371326 3666 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5] pod update took 153.13847ms I0505 19:48:28.371349 3666 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5] addLogicalPort annotation time took 153.162122ms I0505 19:48:28.372268 3666 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5] addLogicalPort took 154.152071ms, libovsdb time 671.331µs I0505 19:48:28.555852 3666 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-dnqkt 8a88a28b611bce95605d9e22499303d3d8dde42377b6947a63d2dd815cf7ddc9 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-dnqkt 8a88a28b611bce95605d9e22499303d3d8dde42377b6947a63d2dd815cf7ddc9 network default NAD default NAD key ] I0505 19:48:28.564111 3666 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5 e802cb4b8ad38f9c416a01d06773ced91817d49c5472c196961994788c93e933 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5 e802cb4b8ad38f9c416a01d06773ced91817d49c5472c196961994788c93e933 network default NAD default NAD key ] I0505 19:48:28.565234 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-dnqkt, hostIfaceName: 8a88a28b611bce9, network: default, NAD default, SandboxID: "8a88a28b611bce95605d9e22499303d3d8dde42377b6947a63d2dd815cf7ddc9", PCI device ID: , UID: "efc43a75-40d1-43d3-ba37-c57d7ff101f0", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0505 19:48:28.573301 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-cdnv5, hostIfaceName: e802cb4b8ad38f9, network: default, NAD default, SandboxID: "e802cb4b8ad38f9c416a01d06773ced91817d49c5472c196961994788c93e933", PCI device ID: , UID: "9487ef81-fc83-4bfa-a38c-fa9b32a56371", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0505 19:48:28.612610 3666 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-dnqkt 8a88a28b611bce95605d9e22499303d3d8dde42377b6947a63d2dd815cf7ddc9 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-dnqkt 8a88a28b611bce95605d9e22499303d3d8dde42377b6947a63d2dd815cf7ddc9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:0f:a2:10:5f:5a\",\"name\":\"8a88a28b611bce9\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e83b9b53-9f86-4f10-9be4-cc0c0263e2a1\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:48:28.628944 3666 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5 e802cb4b8ad38f9c416a01d06773ced91817d49c5472c196961994788c93e933 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-cdnv5 e802cb4b8ad38f9c416a01d06773ced91817d49c5472c196961994788c93e933 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:ef:2b:55:9a:8e\",\"name\":\"e802cb4b8ad38f9\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cd83f2a1-2128-49f6-9693-f4f289b97251\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:48:31.365656 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0505 19:48:31.718441 3666 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0505 19:48:31.719359 3666 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 896.983µs I0505 19:48:31.744996 3666 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0505 19:48:32.391710 3666 namespace.go:98] [openshift-service-ca] adding namespace I0505 19:48:32.392507 3666 namespace.go:102] [openshift-service-ca] adding namespace took 775.599µs I0505 19:48:32.417509 3666 namespace.go:142] [openshift-service-ca] updating namespace I0505 19:48:34.072580 3666 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-np6q5] creating logical port openshift-service-ca_service-ca-d9cb48497-np6q5 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:48:34.072718 3666 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-np6q5 I0505 19:48:34.089784 3666 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-np6q5] pod update took 17.108815ms I0505 19:48:34.089805 3666 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-np6q5] addLogicalPort annotation time took 17.134212ms I0505 19:48:34.090939 3666 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-np6q5] addLogicalPort took 18.372934ms, libovsdb time 825.238µs I0505 19:48:34.428574 3666 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-np6q5 8a9f94e4a87140fce1b629aa1a3de59d35cee9b9b1135e53d28c667d87827be8 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-np6q5 8a9f94e4a87140fce1b629aa1a3de59d35cee9b9b1135e53d28c667d87827be8 network default NAD default NAD key ] I0505 19:48:34.437552 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-np6q5, hostIfaceName: 8a9f94e4a87140f, network: default, NAD default, SandboxID: "8a9f94e4a87140fce1b629aa1a3de59d35cee9b9b1135e53d28c667d87827be8", PCI device ID: , UID: "fb0d1613-b80e-42ec-baab-1f5b72697c15", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0505 19:48:34.483965 3666 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-np6q5 8a9f94e4a87140fce1b629aa1a3de59d35cee9b9b1135e53d28c667d87827be8 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-np6q5 8a9f94e4a87140fce1b629aa1a3de59d35cee9b9b1135e53d28c667d87827be8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:c2:a6:7d:69:b7\",\"name\":\"8a9f94e4a87140f\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/965ded67-cd35-4935-bceb-5704a4c7b573\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:48:40.899883 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0505 19:48:55.276532 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0505 19:48:55.276551 3666 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0505 19:48:55.276556 3666 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default I0505 19:48:55.434277 3666 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-zzwgv] creating logical port openshift-insights_insights-runtime-extractor-zzwgv for pod on switch ip-10-0-2-97.ec2.internal I0505 19:48:55.434360 3666 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-zzwgv I0505 19:48:55.461694 3666 pod.go:62] [openshift-insights/insights-runtime-extractor-zzwgv] pod update took 27.354675ms I0505 19:48:55.461723 3666 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-zzwgv] addLogicalPort annotation time took 27.393292ms I0505 19:48:55.462702 3666 pods.go:271] [openshift-insights/insights-runtime-extractor-zzwgv] addLogicalPort took 28.4427ms, libovsdb time 710.908µs I0505 19:48:55.789595 3666 cni.go:452] [openshift-insights/insights-runtime-extractor-zzwgv 36bea346e1250e72d42beae525419cb053a7de61aa26851d1bc8376e8175acda network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-zzwgv 36bea346e1250e72d42beae525419cb053a7de61aa26851d1bc8376e8175acda network default NAD default NAD key ] I0505 19:48:55.798974 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-zzwgv, hostIfaceName: 36bea346e1250e7, network: default, NAD default, SandboxID: "36bea346e1250e72d42beae525419cb053a7de61aa26851d1bc8376e8175acda", PCI device ID: , UID: "e6aa61d8-bd28-4f19-a041-9d2947bfb685", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0505 19:48:55.844864 3666 cni.go:473] [openshift-insights/insights-runtime-extractor-zzwgv 36bea346e1250e72d42beae525419cb053a7de61aa26851d1bc8376e8175acda network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-zzwgv 36bea346e1250e72d42beae525419cb053a7de61aa26851d1bc8376e8175acda network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:1f:2f:4a:b0:13\",\"name\":\"36bea346e1250e7\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f762869-c080-4c63-a9a2-578bcaa83464\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:48:56.413643 3666 obj_retry.go:498] Detected object openshift-image-registry/image-registry-8648448794-26mqz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:48:56.414197 3666 obj_retry.go:498] Detected object openshift-image-registry/image-registry-8648448794-26mqz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:48:56.435420 3666 obj_retry.go:492] Detected object openshift-image-registry/image-registry-8648448794-26mqz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:48:56.435439 3666 obj_retry.go:492] Detected object openshift-image-registry/image-registry-8648448794-26mqz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:48:56.443779 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-8648448794-26mqz I0505 19:48:56.443800 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-8648448794-26mqz I0505 19:48:57.236943 3666 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5bb69d5545-9x8ks of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:48:57.237592 3666 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5bb69d5545-9x8ks of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:48:57.256312 3666 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5bb69d5545-9x8ks of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:48:57.256330 3666 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5bb69d5545-9x8ks of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:48:57.261004 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5bb69d5545-9x8ks I0505 19:48:57.261052 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5bb69d5545-9x8ks W0505 19:48:57.405292 3666 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0505 19:48:58.274314 3666 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0505 19:49:00.002234 3666 cni.go:452] [openshift-ingress/router-default-7c7b844789-db9r8 4fdbed6c8cfc6d9c72f7c110c450c1da959668e77b0084a652624b25c2c60e2e network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-7c7b844789-db9r8 4fdbed6c8cfc6d9c72f7c110c450c1da959668e77b0084a652624b25c2c60e2e network default NAD default NAD key ] I0505 19:49:00.002281 3666 cni.go:452] [openshift-ingress/router-default-7df9d57-9m8mn 778beab09c127411ff853533bc091db65daddc10b2d747241460f43581b2d944 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-7df9d57-9m8mn 778beab09c127411ff853533bc091db65daddc10b2d747241460f43581b2d944 network default NAD default NAD key ] I0505 19:49:00.014635 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7c7b844789-db9r8, hostIfaceName: 4fdbed6c8cfc6d9, network: default, NAD default, SandboxID: "4fdbed6c8cfc6d9c72f7c110c450c1da959668e77b0084a652624b25c2c60e2e", PCI device ID: , UID: "84fb4e67-9c6d-4a40-8ca7-f2f00f6242d8", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0505 19:49:00.015042 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7df9d57-9m8mn, hostIfaceName: 778beab09c12741, network: default, NAD default, SandboxID: "778beab09c127411ff853533bc091db65daddc10b2d747241460f43581b2d944", PCI device ID: , UID: "8e8e9163-623d-458b-86cb-a0150bd69981", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0505 19:49:00.061227 3666 cni.go:473] [openshift-ingress/router-default-7df9d57-9m8mn 778beab09c127411ff853533bc091db65daddc10b2d747241460f43581b2d944 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-7df9d57-9m8mn 778beab09c127411ff853533bc091db65daddc10b2d747241460f43581b2d944 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:55:d7:0a:bd:ff\",\"name\":\"778beab09c12741\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6b8fe78a-268c-456a-9744-02a9d3857005\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:49:00.075877 3666 cni.go:473] [openshift-ingress/router-default-7c7b844789-db9r8 4fdbed6c8cfc6d9c72f7c110c450c1da959668e77b0084a652624b25c2c60e2e network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-7c7b844789-db9r8 4fdbed6c8cfc6d9c72f7c110c450c1da959668e77b0084a652624b25c2c60e2e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:99:49:fa:56:c7\",\"name\":\"4fdbed6c8cfc6d9\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a7c58bfe-d261-4fb6-9837-ca2e7b6d78fa\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:49:00.779909 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0505 19:49:00.779927 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0505 19:49:00.783983 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 19:49:00.784007 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 19:49:00.784016 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 19:49:00.814881 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0505 19:49:00.814898 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0505 19:49:00.815388 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 19:49:00.815403 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 19:49:00.815411 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 19:49:01.788578 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 19:49:01.788600 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 19:49:01.788644 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0505 19:49:01.788828 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0505 19:49:01.788841 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default I0505 19:49:01.892356 3666 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0505 19:49:01.982356 3666 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0505 19:49:05.275558 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0505 19:49:12.418570 3666 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2642j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:49:12.419104 3666 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2642j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:49:12.419133 3666 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2642j of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:49:12.419143 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0505 19:49:12.419161 3666 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2642j of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:49:12.419170 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0505 19:49:13.471227 3666 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2642j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:49:13.471261 3666 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2642j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:49:13.471276 3666 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2642j of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:49:13.471291 3666 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0d2642j of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:49:14.535139 3666 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:49:14.535272 3666 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0505 19:49:14.558729 3666 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 23.507044ms I0505 19:49:14.558752 3666 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 23.535704ms I0505 19:49:14.559940 3666 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 24.818953ms, libovsdb time 829.6µs I0505 19:49:14.901820 3666 cni.go:452] [openshift-monitoring/alertmanager-main-0 8a2884f85c23422ba91ee79074c88376c508172e459abbe71322ac789698d242 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 8a2884f85c23422ba91ee79074c88376c508172e459abbe71322ac789698d242 network default NAD default NAD key ] I0505 19:49:14.912232 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 8a2884f85c23422, network: default, NAD default, SandboxID: "8a2884f85c23422ba91ee79074c88376c508172e459abbe71322ac789698d242", PCI device ID: , UID: "4fd1bf34-206e-4ce8-9e1c-dbe3587580a5", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0505 19:49:14.963106 3666 cni.go:473] [openshift-monitoring/alertmanager-main-0 8a2884f85c23422ba91ee79074c88376c508172e459abbe71322ac789698d242 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 8a2884f85c23422ba91ee79074c88376c508172e459abbe71322ac789698d242 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:e0:dd:69:b2:10\",\"name\":\"8a2884f85c23422\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2c04f0c9-7bc2-47e2-b97a-01d6ccf2d457\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:49:15.372222 3666 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0505 19:49:15.388983 3666 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0505 19:49:16.858463 3666 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:49:16.858482 3666 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:49:16.858487 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:49:17.559245 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:49:17.559264 3666 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:49:17.559270 3666 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0505 19:49:17.748062 3666 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-82r2b] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-82r2b for pod on switch ip-10-0-2-97.ec2.internal I0505 19:49:17.748214 3666 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-82r2b I0505 19:49:17.763329 3666 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82r2b] pod update took 15.140189ms I0505 19:49:17.763351 3666 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-82r2b] addLogicalPort annotation time took 15.169485ms I0505 19:49:17.764692 3666 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82r2b] addLogicalPort took 16.650779ms, libovsdb time 1.015546ms I0505 19:49:18.110797 3666 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82r2b 1d653f3474501a12211c0162a68a700a54fdf521ee075d19d9cf8f4e89a053fe network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82r2b 1d653f3474501a12211c0162a68a700a54fdf521ee075d19d9cf8f4e89a053fe network default NAD default NAD key ] I0505 19:49:18.121445 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-82r2b, hostIfaceName: 1d653f3474501a1, network: default, NAD default, SandboxID: "1d653f3474501a12211c0162a68a700a54fdf521ee075d19d9cf8f4e89a053fe", PCI device ID: , UID: "9a174862-d2e8-4a78-9505-33259b56377f", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0505 19:49:18.172752 3666 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82r2b 1d653f3474501a12211c0162a68a700a54fdf521ee075d19d9cf8f4e89a053fe network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-82r2b 1d653f3474501a12211c0162a68a700a54fdf521ee075d19d9cf8f4e89a053fe network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:e2:d1:ba:72:5e\",\"name\":\"1d653f3474501a1\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ae253ce2-313b-4035-a601-4cdaa1bf9dc8\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:49:18.425598 3666 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-85cfbc8949-2xzvf] creating logical port openshift-monitoring_metrics-server-85cfbc8949-2xzvf for pod on switch ip-10-0-2-97.ec2.internal I0505 19:49:18.425725 3666 kube.go:256] Updating pod openshift-monitoring/metrics-server-85cfbc8949-2xzvf I0505 19:49:18.444389 3666 pod.go:62] [openshift-monitoring/metrics-server-85cfbc8949-2xzvf] pod update took 18.685066ms I0505 19:49:18.444409 3666 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-85cfbc8949-2xzvf] addLogicalPort annotation time took 18.710209ms I0505 19:49:18.445476 3666 pods.go:271] [openshift-monitoring/metrics-server-85cfbc8949-2xzvf] addLogicalPort took 19.895835ms, libovsdb time 742.997µs I0505 19:49:18.786783 3666 cni.go:452] [openshift-monitoring/metrics-server-85cfbc8949-2xzvf 97313360110c2dbb7432a29287df02a3c3b0e12e55c04f23f0ae4690eab1ad99 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-85cfbc8949-2xzvf 97313360110c2dbb7432a29287df02a3c3b0e12e55c04f23f0ae4690eab1ad99 network default NAD default NAD key ] I0505 19:49:18.796843 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-85cfbc8949-2xzvf, hostIfaceName: 97313360110c2db, network: default, NAD default, SandboxID: "97313360110c2dbb7432a29287df02a3c3b0e12e55c04f23f0ae4690eab1ad99", PCI device ID: , UID: "c5f69711-2aac-479d-80cd-f85045cc8892", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0505 19:49:18.852060 3666 cni.go:473] [openshift-monitoring/metrics-server-85cfbc8949-2xzvf 97313360110c2dbb7432a29287df02a3c3b0e12e55c04f23f0ae4690eab1ad99 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-85cfbc8949-2xzvf 97313360110c2dbb7432a29287df02a3c3b0e12e55c04f23f0ae4690eab1ad99 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:22:03:e9:6b:48\",\"name\":\"97313360110c2db\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/109c1895-f1be-4cbe-886e-a817f9cdc552\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:49:19.885404 3666 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5bb69d5545-xm7sg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:49:19.885442 3666 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5bb69d5545-xm7sg I0505 19:49:19.886784 3666 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5bb69d5545-xm7sg, ips: 10.128.0.5 I0505 19:49:19.886816 3666 obj_retry.go:498] Detected object openshift-image-registry/image-registry-5bb69d5545-xm7sg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:49:19.900048 3666 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5bb69d5545-xm7sg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:49:19.900075 3666 obj_retry.go:492] Detected object openshift-image-registry/image-registry-5bb69d5545-xm7sg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:49:19.909107 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5bb69d5545-xm7sg I0505 19:49:19.909138 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5bb69d5545-xm7sg I0505 19:49:20.784081 3666 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:49:20.784251 3666 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0505 19:49:20.814561 3666 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 30.359973ms I0505 19:49:20.814592 3666 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 30.395134ms I0505 19:49:20.815938 3666 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 31.878197ms, libovsdb time 948.403µs W0505 19:49:20.949451 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0505 19:49:21.148312 3666 cni.go:452] [openshift-monitoring/prometheus-k8s-0 255b8a8221b0df07a0d2701deb35b809cb440ae3e0ac183b8f0c5f42a85a9905 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 255b8a8221b0df07a0d2701deb35b809cb440ae3e0ac183b8f0c5f42a85a9905 network default NAD default NAD key ] I0505 19:49:21.157399 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 255b8a8221b0df0, network: default, NAD default, SandboxID: "255b8a8221b0df07a0d2701deb35b809cb440ae3e0ac183b8f0c5f42a85a9905", PCI device ID: , UID: "01af8d29-7180-4990-8715-b29f3da5616e", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0505 19:49:21.205930 3666 cni.go:473] [openshift-monitoring/prometheus-k8s-0 255b8a8221b0df07a0d2701deb35b809cb440ae3e0ac183b8f0c5f42a85a9905 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 255b8a8221b0df07a0d2701deb35b809cb440ae3e0ac183b8f0c5f42a85a9905 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:05:19:46:90:60\",\"name\":\"255b8a8221b0df0\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/78b58b3c-c5b6-4306-a647-079d07d40574\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:49:21.363023 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0505 19:49:21.602971 3666 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 19:49:21.602988 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 19:49:21.602993 3666 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 19:49:21.602997 3666 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 19:49:21.715872 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:49:21.715898 3666 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:49:21.900880 3666 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:49:21.900896 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:49:22.371000 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 19:49:22.371019 3666 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 19:49:22.371024 3666 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0505 19:49:22.371030 3666 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier I0505 19:49:22.885487 3666 base_network_controller_pods.go:487] [default/openshift-console/console-7c9b4dbb77-xf49l] creating logical port openshift-console_console-7c9b4dbb77-xf49l for pod on switch ip-10-0-2-97.ec2.internal I0505 19:49:22.885595 3666 kube.go:256] Updating pod openshift-console/console-7c9b4dbb77-xf49l I0505 19:49:22.907375 3666 pod.go:62] [openshift-console/console-7c9b4dbb77-xf49l] pod update took 21.808944ms I0505 19:49:22.907401 3666 base_network_controller_pods.go:951] [default/openshift-console/console-7c9b4dbb77-xf49l] addLogicalPort annotation time took 21.835565ms I0505 19:49:22.908729 3666 pods.go:271] [openshift-console/console-7c9b4dbb77-xf49l] addLogicalPort took 23.262534ms, libovsdb time 957.696µs I0505 19:49:23.250036 3666 cni.go:452] [openshift-console/console-7c9b4dbb77-xf49l a682f4a000398b703336105c9e5e7ba8a7d670ff10172d56f95fbd081a3aa45e network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7c9b4dbb77-xf49l a682f4a000398b703336105c9e5e7ba8a7d670ff10172d56f95fbd081a3aa45e network default NAD default NAD key ] I0505 19:49:23.259321 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7c9b4dbb77-xf49l, hostIfaceName: a682f4a000398b7, network: default, NAD default, SandboxID: "a682f4a000398b703336105c9e5e7ba8a7d670ff10172d56f95fbd081a3aa45e", PCI device ID: , UID: "9b86dbfd-a387-4613-9895-3ff0814e0a3b", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0505 19:49:23.310639 3666 cni.go:473] [openshift-console/console-7c9b4dbb77-xf49l a682f4a000398b703336105c9e5e7ba8a7d670ff10172d56f95fbd081a3aa45e network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7c9b4dbb77-xf49l a682f4a000398b703336105c9e5e7ba8a7d670ff10172d56f95fbd081a3aa45e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:55:65:e2:df:54\",\"name\":\"a682f4a000398b7\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/da24c95a-9531-489f-b5db-ff1df959b027\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:49:27.946505 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0505 19:49:29.385617 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0505 19:49:31.217371 3666 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0505 19:49:31.395667 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0505 19:49:32.601926 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0505 19:49:33.930340 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0505 19:49:36.937552 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0505 19:49:36.937673 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0505 19:49:36.937738 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0505 19:49:37.949294 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0505 19:49:42.367506 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0505 19:49:42.367543 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0505 19:49:42.875840 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0505 19:49:44.963379 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:49:44.963398 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:49:44.963403 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:49:47.982872 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0505 19:49:47.982869 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0505 19:49:47.983045 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0505 19:49:48.985684 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0505 19:49:51.677486 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0505 19:49:56.692091 3666 obj_retry.go:498] Detected object openshift-console/console-7c9b4dbb77-r84lg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:49:56.692688 3666 obj_retry.go:498] Detected object openshift-console/console-7c9b4dbb77-r84lg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:49:57.484545 3666 obj_retry.go:492] Detected object openshift-console/console-7c9b4dbb77-r84lg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:49:57.484566 3666 obj_retry.go:492] Detected object openshift-console/console-7c9b4dbb77-r84lg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:49:57.492823 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7c9b4dbb77-r84lg I0505 19:49:57.492856 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7c9b4dbb77-r84lg I0505 19:50:01.061440 3666 cni.go:452] [openshift-console/console-7c9b4dbb77-xf49l a682f4a000398b703336105c9e5e7ba8a7d670ff10172d56f95fbd081a3aa45e network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7c9b4dbb77-xf49l a682f4a000398b703336105c9e5e7ba8a7d670ff10172d56f95fbd081a3aa45e network default NAD default NAD key ] I0505 19:50:01.143499 3666 cni.go:473] [openshift-console/console-7c9b4dbb77-xf49l a682f4a000398b703336105c9e5e7ba8a7d670ff10172d56f95fbd081a3aa45e network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7c9b4dbb77-xf49l a682f4a000398b703336105c9e5e7ba8a7d670ff10172d56f95fbd081a3aa45e network default NAD default NAD key default], result "{}", err I0505 19:50:01.159621 3666 cni.go:452] [openshift-multus/network-metrics-daemon-l5n5q 57be030fcca7762e5baae8f005f4f9318004f0675bad4d6435d8052154bca2f8 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-l5n5q 57be030fcca7762e5baae8f005f4f9318004f0675bad4d6435d8052154bca2f8 network default NAD default NAD key ] I0505 19:50:01.168941 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-l5n5q, hostIfaceName: 57be030fcca7762, network: default, NAD default, SandboxID: "57be030fcca7762e5baae8f005f4f9318004f0675bad4d6435d8052154bca2f8", PCI device ID: , UID: "ba35fd05-c2f7-4b8b-aa33-93eb5c1eee9e", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0505 19:50:01.195602 3666 obj_retry.go:498] Detected object openshift-console/console-7c9b4dbb77-xf49l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:01.195638 3666 pods.go:176] Deleting pod: openshift-console/console-7c9b4dbb77-xf49l I0505 19:50:01.197129 3666 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7c9b4dbb77-xf49l, ips: 10.128.0.19 I0505 19:50:01.197165 3666 obj_retry.go:498] Detected object openshift-console/console-7c9b4dbb77-xf49l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:01.216605 3666 cni.go:473] [openshift-multus/network-metrics-daemon-l5n5q 57be030fcca7762e5baae8f005f4f9318004f0675bad4d6435d8052154bca2f8 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-l5n5q 57be030fcca7762e5baae8f005f4f9318004f0675bad4d6435d8052154bca2f8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:fd:f3:d6:2c:72\",\"name\":\"57be030fcca7762\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3747a1c0-12ce-4485-8797-a4c66f1ea137\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:50:02.032938 3666 obj_retry.go:492] Detected object openshift-console/console-7c9b4dbb77-xf49l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:02.032962 3666 obj_retry.go:492] Detected object openshift-console/console-7c9b4dbb77-xf49l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:02.043712 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7c9b4dbb77-xf49l I0505 19:50:02.043742 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7c9b4dbb77-xf49l I0505 19:50:17.226781 3666 cni.go:452] [openshift-ingress/router-default-7c7b844789-db9r8 4fdbed6c8cfc6d9c72f7c110c450c1da959668e77b0084a652624b25c2c60e2e network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-7c7b844789-db9r8 4fdbed6c8cfc6d9c72f7c110c450c1da959668e77b0084a652624b25c2c60e2e network default NAD default NAD key ] I0505 19:50:17.304197 3666 cni.go:473] [openshift-ingress/router-default-7c7b844789-db9r8 4fdbed6c8cfc6d9c72f7c110c450c1da959668e77b0084a652624b25c2c60e2e network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-7c7b844789-db9r8 4fdbed6c8cfc6d9c72f7c110c450c1da959668e77b0084a652624b25c2c60e2e network default NAD default NAD key default], result "{}", err I0505 19:50:17.355043 3666 obj_retry.go:498] Detected object openshift-ingress/router-default-7c7b844789-db9r8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:17.355074 3666 pods.go:176] Deleting pod: openshift-ingress/router-default-7c7b844789-db9r8 I0505 19:50:17.356317 3666 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-7c7b844789-db9r8, ips: 10.128.0.9 I0505 19:50:17.356348 3666 obj_retry.go:498] Detected object openshift-ingress/router-default-7c7b844789-db9r8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:18.106074 3666 obj_retry.go:492] Detected object openshift-ingress/router-default-7c7b844789-db9r8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:18.106100 3666 obj_retry.go:492] Detected object openshift-ingress/router-default-7c7b844789-db9r8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:18.106121 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7c7b844789-db9r8 I0505 19:50:18.106132 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7c7b844789-db9r8 I0505 19:50:21.717909 3666 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29633510-cmth5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:21.718443 3666 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29633510-cmth5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0505 19:50:21.833668 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:50:21.833695 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:50:21.833710 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:50:21.833717 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:50:22.150470 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:50:22.150497 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0505 19:50:22.742781 3666 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29633510-cmth5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:22.742803 3666 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29633510-cmth5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:30.853986 3666 cni.go:452] [openshift-dns/dns-default-8sgmm c7bcc8a6798e324c9379c6bc5e3821f025b57dffdeb443da3f727b26bd988217 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-8sgmm c7bcc8a6798e324c9379c6bc5e3821f025b57dffdeb443da3f727b26bd988217 network default NAD default NAD key ] I0505 19:50:30.863209 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-8sgmm, hostIfaceName: c7bcc8a6798e324, network: default, NAD default, SandboxID: "c7bcc8a6798e324c9379c6bc5e3821f025b57dffdeb443da3f727b26bd988217", PCI device ID: , UID: "f35e7daf-0ae2-421b-9f9f-f493fc5f8917", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0505 19:50:30.913237 3666 cni.go:473] [openshift-dns/dns-default-8sgmm c7bcc8a6798e324c9379c6bc5e3821f025b57dffdeb443da3f727b26bd988217 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-8sgmm c7bcc8a6798e324c9379c6bc5e3821f025b57dffdeb443da3f727b26bd988217 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:8e:90:45:92:14\",\"name\":\"c7bcc8a6798e324\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/63df3755-75b9-4633-b0f7-e7b655af2253\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:50:34.885257 3666 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0505 19:50:36.392094 3666 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:36.392677 3666 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:36.880797 3666 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:36.880819 3666 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:36.891297 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0505 19:50:36.891313 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0505 19:50:39.262986 3666 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-76b4f48bcb-274md] creating logical port openshift-monitoring_telemeter-client-76b4f48bcb-274md for pod on switch ip-10-0-2-97.ec2.internal I0505 19:50:39.263104 3666 kube.go:256] Updating pod openshift-monitoring/telemeter-client-76b4f48bcb-274md I0505 19:50:39.283593 3666 pod.go:62] [openshift-monitoring/telemeter-client-76b4f48bcb-274md] pod update took 20.526081ms I0505 19:50:39.283617 3666 base_network_controller_pods.go:951] [default/openshift-monitoring/telemeter-client-76b4f48bcb-274md] addLogicalPort annotation time took 20.559263ms I0505 19:50:39.284801 3666 pods.go:271] [openshift-monitoring/telemeter-client-76b4f48bcb-274md] addLogicalPort took 21.838234ms, libovsdb time 848.87µs I0505 19:50:39.623099 3666 cni.go:452] [openshift-monitoring/telemeter-client-76b4f48bcb-274md cdfb0c3e65c6b64f10759c2b49b820dd87e4f3f62068d344204d0f5ddf7c5730 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/telemeter-client-76b4f48bcb-274md cdfb0c3e65c6b64f10759c2b49b820dd87e4f3f62068d344204d0f5ddf7c5730 network default NAD default NAD key ] I0505 19:50:39.632263 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-76b4f48bcb-274md, hostIfaceName: cdfb0c3e65c6b64, network: default, NAD default, SandboxID: "cdfb0c3e65c6b64f10759c2b49b820dd87e4f3f62068d344204d0f5ddf7c5730", PCI device ID: , UID: "642f1ce7-1818-4ed9-871c-df1ef3d324fa", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0505 19:50:39.683802 3666 cni.go:473] [openshift-monitoring/telemeter-client-76b4f48bcb-274md cdfb0c3e65c6b64f10759c2b49b820dd87e4f3f62068d344204d0f5ddf7c5730 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/telemeter-client-76b4f48bcb-274md cdfb0c3e65c6b64f10759c2b49b820dd87e4f3f62068d344204d0f5ddf7c5730 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:88:c3:f6:c2:f1\",\"name\":\"cdfb0c3e65c6b64\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/649632bc-c9aa-453a-8cbd-4da69037c61e\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:50:40.905451 3666 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:40.906007 3666 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:41.207153 3666 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:41.207182 3666 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:41.215950 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0505 19:50:41.215978 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0505 19:50:42.454066 3666 base_network_controller_pods.go:487] [default/openshift-console/console-7467788994-pk2hn] creating logical port openshift-console_console-7467788994-pk2hn for pod on switch ip-10-0-2-97.ec2.internal I0505 19:50:42.454175 3666 kube.go:256] Updating pod openshift-console/console-7467788994-pk2hn I0505 19:50:42.482886 3666 pod.go:62] [openshift-console/console-7467788994-pk2hn] pod update took 28.742853ms I0505 19:50:42.482914 3666 base_network_controller_pods.go:951] [default/openshift-console/console-7467788994-pk2hn] addLogicalPort annotation time took 28.778083ms I0505 19:50:42.484049 3666 pods.go:271] [openshift-console/console-7467788994-pk2hn] addLogicalPort took 30.006379ms, libovsdb time 835.788µs I0505 19:50:42.815222 3666 cni.go:452] [openshift-console/console-7467788994-pk2hn 176bd9e87c1626eae28955fa5009f78211a9512b4fe618a1344b2dd3c20dd54e network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7467788994-pk2hn 176bd9e87c1626eae28955fa5009f78211a9512b4fe618a1344b2dd3c20dd54e network default NAD default NAD key ] I0505 19:50:42.827728 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7467788994-pk2hn, hostIfaceName: 176bd9e87c1626e, network: default, NAD default, SandboxID: "176bd9e87c1626eae28955fa5009f78211a9512b4fe618a1344b2dd3c20dd54e", PCI device ID: , UID: "482e2f13-c4fe-4db3-8399-5d3c89f24dad", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0505 19:50:42.879211 3666 cni.go:473] [openshift-console/console-7467788994-pk2hn 176bd9e87c1626eae28955fa5009f78211a9512b4fe618a1344b2dd3c20dd54e network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7467788994-pk2hn 176bd9e87c1626eae28955fa5009f78211a9512b4fe618a1344b2dd3c20dd54e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:a2:df:aa:81:a8\",\"name\":\"176bd9e87c1626e\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ce0f9850-3106-4787-9db0-9191c9c4fb0b\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:50:43.185300 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0505 19:50:43.185328 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0505 19:50:43.185337 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0505 19:50:53.244555 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0505 19:50:58.518554 3666 cni.go:452] [openshift-monitoring/alertmanager-main-0 8a2884f85c23422ba91ee79074c88376c508172e459abbe71322ac789698d242 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 8a2884f85c23422ba91ee79074c88376c508172e459abbe71322ac789698d242 network default NAD default NAD key ] I0505 19:50:58.594745 3666 cni.go:473] [openshift-monitoring/alertmanager-main-0 8a2884f85c23422ba91ee79074c88376c508172e459abbe71322ac789698d242 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 8a2884f85c23422ba91ee79074c88376c508172e459abbe71322ac789698d242 network default NAD default NAD key default], result "{}", err I0505 19:50:58.657932 3666 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:58.657970 3666 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0505 19:50:58.659973 3666 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.128.0.15 I0505 19:50:58.660011 3666 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:50:59.261971 3666 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:59.262001 3666 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:50:59.271531 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0505 19:50:59.271559 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0505 19:50:59.310722 3666 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:50:59.310858 3666 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0505 19:50:59.336860 3666 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 26.03825ms I0505 19:50:59.336885 3666 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 26.072344ms I0505 19:50:59.338033 3666 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 27.325551ms, libovsdb time 860.537µs I0505 19:50:59.669996 3666 cni.go:452] [openshift-monitoring/alertmanager-main-0 bc6545f9dc22c11e799792ee36cea25b80ace512b406bb623d82d9ee53768298 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 bc6545f9dc22c11e799792ee36cea25b80ace512b406bb623d82d9ee53768298 network default NAD default NAD key ] I0505 19:50:59.679597 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: bc6545f9dc22c11, network: default, NAD default, SandboxID: "bc6545f9dc22c11e799792ee36cea25b80ace512b406bb623d82d9ee53768298", PCI device ID: , UID: "6e323d3b-b7a2-4c2f-bc2b-f74073c38570", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0505 19:50:59.728477 3666 cni.go:473] [openshift-monitoring/alertmanager-main-0 bc6545f9dc22c11e799792ee36cea25b80ace512b406bb623d82d9ee53768298 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 bc6545f9dc22c11e799792ee36cea25b80ace512b406bb623d82d9ee53768298 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:9a:85:83:e8:17\",\"name\":\"bc6545f9dc22c11\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3b3771b2-2f51-472b-9ff3-03e292c98f01\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:51:07.743996 3666 obj_retry.go:498] Detected object openshift-console/console-55f4987765-nckvx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:51:07.744543 3666 obj_retry.go:498] Detected object openshift-console/console-55f4987765-nckvx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:51:08.056374 3666 obj_retry.go:492] Detected object openshift-console/console-55f4987765-nckvx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:51:08.056400 3666 obj_retry.go:492] Detected object openshift-console/console-55f4987765-nckvx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:51:08.060384 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-55f4987765-nckvx I0505 19:51:08.060404 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-55f4987765-nckvx I0505 19:51:18.592040 3666 obj_retry.go:498] Detected object openshift-console/console-55f4987765-9d4zn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:51:18.592662 3666 obj_retry.go:498] Detected object openshift-console/console-55f4987765-9d4zn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:51:19.082436 3666 obj_retry.go:492] Detected object openshift-console/console-55f4987765-9d4zn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:51:19.082460 3666 obj_retry.go:492] Detected object openshift-console/console-55f4987765-9d4zn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:51:19.091379 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-55f4987765-9d4zn I0505 19:51:19.091397 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-55f4987765-9d4zn W0505 19:51:29.732468 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:51:29.732487 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0505 19:51:29.732492 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0505 19:51:40.402509 3666 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0505 19:51:43.383611 3666 cni.go:452] [openshift-monitoring/prometheus-k8s-0 255b8a8221b0df07a0d2701deb35b809cb440ae3e0ac183b8f0c5f42a85a9905 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 255b8a8221b0df07a0d2701deb35b809cb440ae3e0ac183b8f0c5f42a85a9905 network default NAD default NAD key ] I0505 19:51:43.452552 3666 cni.go:473] [openshift-monitoring/prometheus-k8s-0 255b8a8221b0df07a0d2701deb35b809cb440ae3e0ac183b8f0c5f42a85a9905 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 255b8a8221b0df07a0d2701deb35b809cb440ae3e0ac183b8f0c5f42a85a9905 network default NAD default NAD key default], result "{}", err I0505 19:51:43.518979 3666 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:51:43.519018 3666 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0505 19:51:43.520424 3666 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.128.0.18 I0505 19:51:43.520459 3666 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:51:44.457655 3666 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:51:44.457676 3666 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:51:44.472471 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0505 19:51:44.472502 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0505 19:51:44.517893 3666 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:51:44.518025 3666 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0505 19:51:44.542553 3666 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 24.58156ms I0505 19:51:44.542574 3666 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 24.608089ms I0505 19:51:44.543763 3666 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 25.889284ms, libovsdb time 829.372µs I0505 19:51:44.879386 3666 cni.go:452] [openshift-monitoring/prometheus-k8s-0 a1dcf6939d5809af7547dbeb0965a1cdb18f371cf0c75dcc26071c9e21df325d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 a1dcf6939d5809af7547dbeb0965a1cdb18f371cf0c75dcc26071c9e21df325d network default NAD default NAD key ] I0505 19:51:44.888758 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: a1dcf6939d5809a, network: default, NAD default, SandboxID: "a1dcf6939d5809af7547dbeb0965a1cdb18f371cf0c75dcc26071c9e21df325d", PCI device ID: , UID: "9d6b6bad-0701-4f89-ab7a-fbda401aed9d", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0505 19:51:44.936811 3666 cni.go:473] [openshift-monitoring/prometheus-k8s-0 a1dcf6939d5809af7547dbeb0965a1cdb18f371cf0c75dcc26071c9e21df325d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 a1dcf6939d5809af7547dbeb0965a1cdb18f371cf0c75dcc26071c9e21df325d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:b4:b0:6e:66:90\",\"name\":\"a1dcf6939d5809a\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f0dbfdaf-c07c-45ad-aae0-ab67dce9ebc1\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:51:46.419470 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0505 19:51:59.322656 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=12 I0505 19:52:09.322095 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0505 19:52:31.430204 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0505 19:52:35.329786 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=606 W0505 19:52:45.732565 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0505 19:52:45.732590 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0505 19:52:53.424733 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0505 19:52:56.441905 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0505 19:52:58.442277 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0505 19:53:38.406949 3666 obj_retry.go:498] Detected object openshift-console/console-7467788994-jm257 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:53:38.407531 3666 obj_retry.go:498] Detected object openshift-console/console-7467788994-jm257 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:53:39.319721 3666 obj_retry.go:492] Detected object openshift-console/console-7467788994-jm257 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:53:39.319741 3666 obj_retry.go:492] Detected object openshift-console/console-7467788994-jm257 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:53:39.324289 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7467788994-jm257 I0505 19:53:39.324314 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7467788994-jm257 I0505 19:53:48.809201 3666 cni.go:452] [openshift-console/console-7467788994-pk2hn 176bd9e87c1626eae28955fa5009f78211a9512b4fe618a1344b2dd3c20dd54e network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-7467788994-pk2hn 176bd9e87c1626eae28955fa5009f78211a9512b4fe618a1344b2dd3c20dd54e network default NAD default NAD key ] I0505 19:53:48.887566 3666 cni.go:473] [openshift-console/console-7467788994-pk2hn 176bd9e87c1626eae28955fa5009f78211a9512b4fe618a1344b2dd3c20dd54e network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-7467788994-pk2hn 176bd9e87c1626eae28955fa5009f78211a9512b4fe618a1344b2dd3c20dd54e network default NAD default NAD key default], result "{}", err I0505 19:53:48.999411 3666 obj_retry.go:498] Detected object openshift-console/console-7467788994-pk2hn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:53:48.999447 3666 pods.go:176] Deleting pod: openshift-console/console-7467788994-pk2hn I0505 19:53:49.000661 3666 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7467788994-pk2hn, ips: 10.128.0.21 I0505 19:53:49.000684 3666 obj_retry.go:498] Detected object openshift-console/console-7467788994-pk2hn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:53:49.929534 3666 obj_retry.go:492] Detected object openshift-console/console-7467788994-pk2hn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:53:49.929572 3666 obj_retry.go:492] Detected object openshift-console/console-7467788994-pk2hn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:53:49.935081 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7467788994-pk2hn I0505 19:53:49.935115 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7467788994-pk2hn I0505 19:54:04.323452 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=72 W0505 19:54:19.469331 3666 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0505 19:54:30.531004 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0505 19:54:30.531089 3666 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-t2929 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-t2929: no pod IPs found I0505 19:54:31.436883 3666 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0505 19:54:42.407969 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0505 19:54:51.323111 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=92 I0505 19:55:02.414108 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0505 19:55:39.322764 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=145 I0505 19:56:00.403561 3666 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 I0505 19:56:02.633067 3666 namespace.go:98] [tsf] adding namespace I0505 19:56:02.634017 3666 namespace.go:102] [tsf] adding namespace took 927.042µs I0505 19:56:02.657781 3666 namespace.go:142] [tsf] updating namespace I0505 19:56:02.675220 3666 namespace.go:142] [tsf] updating namespace I0505 19:56:06.958266 3666 namespace.go:98] [tsf-keycloak] adding namespace I0505 19:56:06.958708 3666 namespace.go:98] [konflux-ui] adding namespace I0505 19:56:06.959142 3666 namespace.go:102] [tsf-keycloak] adding namespace took 856.927µs I0505 19:56:06.959842 3666 namespace.go:102] [konflux-ui] adding namespace took 1.11838ms I0505 19:56:06.961487 3666 namespace.go:98] [konflux-operator] adding namespace I0505 19:56:06.961569 3666 namespace.go:98] [tsf-tpa] adding namespace I0505 19:56:06.962095 3666 namespace.go:102] [konflux-operator] adding namespace took 591.555µs I0505 19:56:06.962681 3666 namespace.go:102] [tsf-tpa] adding namespace took 1.093485ms I0505 19:56:06.975123 3666 namespace.go:98] [cert-manager-operator] adding namespace I0505 19:56:06.975223 3666 namespace.go:98] [tsf-tas] adding namespace I0505 19:56:06.975322 3666 namespace.go:98] [rhbk-operator] adding namespace I0505 19:56:06.975761 3666 namespace.go:102] [cert-manager-operator] adding namespace took 622.783µs I0505 19:56:06.976388 3666 namespace.go:102] [tsf-tas] adding namespace took 1.154069ms I0505 19:56:06.976914 3666 namespace.go:102] [rhbk-operator] adding namespace took 1.578915ms I0505 19:56:06.976944 3666 namespace.go:98] [rhtpa-operator] adding namespace I0505 19:56:06.977378 3666 namespace.go:102] [rhtpa-operator] adding namespace took 426.961µs I0505 19:56:06.992373 3666 namespace.go:142] [tsf-keycloak] updating namespace I0505 19:56:07.023548 3666 namespace.go:142] [tsf-keycloak] updating namespace I0505 19:56:07.031203 3666 namespace.go:142] [cert-manager-operator] updating namespace I0505 19:56:07.031329 3666 namespace.go:142] [konflux-ui] updating namespace I0505 19:56:07.031440 3666 namespace.go:142] [tsf-tpa] updating namespace I0505 19:56:07.045069 3666 namespace.go:142] [konflux-operator] updating namespace I0505 19:56:07.057013 3666 namespace.go:142] [rhbk-operator] updating namespace I0505 19:56:07.057027 3666 namespace.go:142] [tsf-tas] updating namespace I0505 19:56:07.057173 3666 namespace.go:142] [rhtpa-operator] updating namespace I0505 19:56:07.061760 3666 namespace.go:142] [tsf-keycloak] updating namespace I0505 19:56:07.150216 3666 namespace.go:142] [tsf-tpa] updating namespace I0505 19:56:07.177100 3666 namespace.go:142] [tsf-tpa] updating namespace I0505 19:56:07.180061 3666 namespace.go:142] [cert-manager-operator] updating namespace I0505 19:56:07.201111 3666 namespace.go:142] [cert-manager-operator] updating namespace I0505 19:56:07.220116 3666 namespace.go:142] [tsf-tas] updating namespace I0505 19:56:07.241006 3666 namespace.go:142] [tsf-tas] updating namespace I0505 19:56:07.252170 3666 namespace.go:142] [rhbk-operator] updating namespace I0505 19:56:07.277500 3666 namespace.go:142] [rhbk-operator] updating namespace I0505 19:56:07.293037 3666 namespace.go:142] [rhtpa-operator] updating namespace I0505 19:56:07.317763 3666 namespace.go:142] [rhtpa-operator] updating namespace I0505 19:56:07.337042 3666 namespace.go:142] [konflux-ui] updating namespace I0505 19:56:07.354230 3666 namespace.go:142] [konflux-ui] updating namespace I0505 19:56:07.389412 3666 namespace.go:142] [konflux-operator] updating namespace I0505 19:56:07.406859 3666 namespace.go:142] [konflux-operator] updating namespace I0505 19:56:08.497244 3666 namespace.go:142] [tsf-tpa] updating namespace I0505 19:56:08.512657 3666 namespace.go:142] [tsf-keycloak] updating namespace I0505 19:56:08.795813 3666 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-2-97.ec2.internal I0505 19:56:08.795933 3666 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0505 19:56:08.824568 3666 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 28.665349ms I0505 19:56:08.824591 3666 base_network_controller_pods.go:951] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 28.694293ms I0505 19:56:08.826012 3666 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 30.21137ms, libovsdb time 986.056µs I0505 19:56:09.164384 3666 cni.go:452] [tsf/test-tsf-subscriptions 44714b32dec065550964e6143e02500a9a9e22789c6e6e32aff0e6f81401d5db network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions 44714b32dec065550964e6143e02500a9a9e22789c6e6e32aff0e6f81401d5db network default NAD default NAD key ] I0505 19:56:09.173788 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 44714b32dec0655, network: default, NAD default, SandboxID: "44714b32dec065550964e6143e02500a9a9e22789c6e6e32aff0e6f81401d5db", PCI device ID: , UID: "882c1a80-4f57-4e99-bc9e-362d1b6de762", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0505 19:56:09.221462 3666 cni.go:473] [tsf/test-tsf-subscriptions 44714b32dec065550964e6143e02500a9a9e22789c6e6e32aff0e6f81401d5db network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions 44714b32dec065550964e6143e02500a9a9e22789c6e6e32aff0e6f81401d5db network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:17:a4:82:1a:89\",\"name\":\"44714b32dec0655\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9fcaf6f4-d2f3-4fbc-a05f-dc095405a7ba\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:56:15.008142 3666 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136nzq2q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:15.008681 3666 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136nzq2q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:15.008711 3666 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136nzq2q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:15.008720 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 19:56:15.008729 3666 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136nzq2q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:15.008734 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 19:56:16.014908 3666 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hsjpv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:16.015491 3666 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hsjpv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:16.015535 3666 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hsjpv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:16.015548 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 19:56:16.015559 3666 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hsjpv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:16.015563 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 19:56:16.036844 3666 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136nzq2q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:16.036866 3666 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136nzq2q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:16.036881 3666 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136nzq2q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:16.036891 3666 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136nzq2q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:17.046625 3666 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hsjpv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:17.046652 3666 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hsjpv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:17.046669 3666 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hsjpv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:17.046680 3666 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7hsjpv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:28.116925 3666 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6en47l5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:28.117420 3666 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6en47l5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:28.117449 3666 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6en47l5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:28.117461 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 19:56:28.117474 3666 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6en47l5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:28.117479 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 19:56:28.144685 3666 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51m448p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:28.145084 3666 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51m448p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:28.145109 3666 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51m448p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:28.145117 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 19:56:28.145125 3666 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51m448p of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:28.145129 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 19:56:29.144893 3666 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6en47l5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:29.144946 3666 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6en47l5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:29.144967 3666 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6en47l5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:29.144977 3666 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6en47l5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:29.176140 3666 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51m448p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:29.176165 3666 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51m448p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:29.176179 3666 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51m448p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:29.176190 3666 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51m448p of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:30.093228 3666 namespace.go:142] [openshift-operators] updating namespace I0505 19:56:30.114114 3666 namespace.go:142] [openshift-operators] updating namespace I0505 19:56:31.150360 3666 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26v45m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:31.150864 3666 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26v45m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:31.150892 3666 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26v45m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:31.150901 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 19:56:31.150910 3666 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26v45m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:31.150914 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 19:56:32.119755 3666 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d58sf6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:32.120300 3666 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d58sf6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:56:32.120339 3666 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d58sf6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:32.120351 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0505 19:56:32.120363 3666 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d58sf6 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0505 19:56:32.120368 3666 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0505 19:56:32.180573 3666 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26v45m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:32.180593 3666 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26v45m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:32.180605 3666 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26v45m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:32.180615 3666 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf26v45m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:33.159217 3666 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d58sf6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:33.159240 3666 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d58sf6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:33.159253 3666 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d58sf6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:33.159265 3666 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d58sf6 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:56:37.330375 3666 namespace.go:142] [openshift-operators] updating namespace W0505 19:56:45.087298 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0505 19:56:45.345770 3666 namespace.go:98] [trusted-artifact-signer] adding namespace I0505 19:56:45.346616 3666 namespace.go:102] [trusted-artifact-signer] adding namespace took 823.329µs I0505 19:56:45.365269 3666 namespace.go:142] [trusted-artifact-signer] updating namespace I0505 19:56:45.385578 3666 namespace.go:142] [trusted-artifact-signer] updating namespace W0505 19:56:50.340905 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0505 19:56:50.411139 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0505 19:56:54.423946 3666 namespace.go:98] [openshift-pipelines] adding namespace I0505 19:56:54.424736 3666 namespace.go:102] [openshift-pipelines] adding namespace took 769.651µs I0505 19:56:54.451463 3666 namespace.go:142] [openshift-pipelines] updating namespace I0505 19:56:55.445076 3666 namespace.go:98] [cert-manager] adding namespace I0505 19:56:55.445940 3666 namespace.go:102] [cert-manager] adding namespace took 838.617µs I0505 19:56:55.473084 3666 namespace.go:142] [cert-manager] updating namespace I0505 19:56:55.507081 3666 namespace.go:142] [cert-manager] updating namespace I0505 19:56:55.942159 3666 namespace.go:142] [cert-manager] updating namespace I0505 19:56:56.147636 3666 namespace.go:142] [cert-manager-operator] updating namespace I0505 19:56:57.786233 3666 namespace.go:142] [konflux-operator] updating namespace I0505 19:56:58.036463 3666 namespace.go:142] [konflux-ui] updating namespace I0505 19:56:58.241981 3666 namespace.go:142] [open-cluster-management-2q3veqdi3e5ncvq7hjcncpt81jc24hvs] updating namespace I0505 19:56:58.648245 3666 namespace.go:142] [rhbk-operator] updating namespace I0505 19:56:58.921111 3666 namespace.go:142] [rhtpa-operator] updating namespace I0505 19:56:59.240434 3666 namespace.go:142] [trusted-artifact-signer] updating namespace I0505 19:56:59.354613 3666 namespace.go:142] [tsf] updating namespace W0505 19:56:59.415168 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0505 19:56:59.443914 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0505 19:56:59.469103 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0505 19:56:59.500121 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0505 19:56:59.708586 3666 namespace.go:142] [tsf-keycloak] updating namespace I0505 19:56:59.927388 3666 namespace.go:142] [tsf-tas] updating namespace I0505 19:57:00.183083 3666 namespace.go:142] [tsf-tpa] updating namespace I0505 19:57:00.359841 3666 namespace.go:142] [cert-manager] updating namespace I0505 19:57:00.379326 3666 namespace.go:142] [cert-manager-operator] updating namespace I0505 19:57:00.396375 3666 namespace.go:142] [default] updating namespace I0505 19:57:00.416620 3666 namespace.go:142] [konflux-operator] updating namespace I0505 19:57:00.432466 3666 namespace.go:142] [konflux-ui] updating namespace I0505 19:57:00.450583 3666 namespace.go:142] [open-cluster-management-2q3veqdi3e5ncvq7hjcncpt81jc24hvs] updating namespace I0505 19:57:00.462866 3666 namespace.go:142] [openshift] updating namespace I0505 19:57:00.487721 3666 namespace.go:142] [rhbk-operator] updating namespace I0505 19:57:00.506480 3666 namespace.go:142] [rhtpa-operator] updating namespace I0505 19:57:00.518442 3666 namespace.go:142] [trusted-artifact-signer] updating namespace I0505 19:57:00.533440 3666 namespace.go:142] [tsf] updating namespace I0505 19:57:00.559194 3666 namespace.go:142] [tsf-keycloak] updating namespace I0505 19:57:00.570874 3666 namespace.go:142] [tsf-tas] updating namespace I0505 19:57:00.596955 3666 namespace.go:142] [tsf-tpa] updating namespace I0505 19:57:00.672339 3666 namespace.go:142] [cert-manager] updating namespace I0505 19:57:00.778002 3666 namespace.go:142] [cert-manager-operator] updating namespace I0505 19:57:00.861775 3666 namespace.go:142] [default] updating namespace I0505 19:57:00.931378 3666 namespace.go:142] [konflux-operator] updating namespace I0505 19:57:00.996451 3666 namespace.go:142] [konflux-ui] updating namespace I0505 19:57:01.090033 3666 namespace.go:142] [open-cluster-management-2q3veqdi3e5ncvq7hjcncpt81jc24hvs] updating namespace I0505 19:57:01.256811 3666 namespace.go:142] [openshift] updating namespace I0505 19:57:01.429283 3666 namespace.go:142] [rhbk-operator] updating namespace I0505 19:57:01.569869 3666 namespace.go:142] [rhtpa-operator] updating namespace I0505 19:57:01.757828 3666 namespace.go:142] [trusted-artifact-signer] updating namespace I0505 19:57:01.864729 3666 namespace.go:142] [tsf] updating namespace I0505 19:57:02.008143 3666 namespace.go:142] [tsf-keycloak] updating namespace I0505 19:57:02.221137 3666 namespace.go:142] [tsf-tas] updating namespace I0505 19:57:02.337529 3666 namespace.go:142] [tsf-tpa] updating namespace I0505 19:57:03.716170 3666 namespace.go:142] [openshift-pipelines] updating namespace W0505 19:57:04.794066 3666 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0505 19:57:04.794093 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook I0505 19:57:17.292981 3666 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-m2ltd] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-m2ltd for pod on switch ip-10-0-2-97.ec2.internal I0505 19:57:17.293075 3666 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-m2ltd I0505 19:57:17.319659 3666 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-m2ltd] pod update took 26.612871ms I0505 19:57:17.319684 3666 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-m2ltd] addLogicalPort annotation time took 26.642954ms I0505 19:57:17.320861 3666 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-m2ltd] addLogicalPort took 27.898642ms, libovsdb time 832.929µs I0505 19:57:17.651562 3666 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-m2ltd d4bee81a2cac81f81d4c707372c948a7c4fe6de866dc28f5055071c78a19ad58 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-m2ltd d4bee81a2cac81f81d4c707372c948a7c4fe6de866dc28f5055071c78a19ad58 network default NAD default NAD key ] I0505 19:57:17.661006 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-m2ltd, hostIfaceName: d4bee81a2cac81f, network: default, NAD default, SandboxID: "d4bee81a2cac81f81d4c707372c948a7c4fe6de866dc28f5055071c78a19ad58", PCI device ID: , UID: "f47b3a36-ecb9-4d20-9cba-dfcc42453051", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0505 19:57:17.710199 3666 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-m2ltd d4bee81a2cac81f81d4c707372c948a7c4fe6de866dc28f5055071c78a19ad58 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-m2ltd d4bee81a2cac81f81d4c707372c948a7c4fe6de866dc28f5055071c78a19ad58 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:8b:e7:de:dd:1b\",\"name\":\"d4bee81a2cac81f\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fbfbb1ed-04b0-429f-9930-56a057592152\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:57:20.757265 3666 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 19:57:20.757285 3666 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 19:57:20.757289 3666 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0505 19:57:21.738722 3666 cni.go:452] [tsf/test-tsf-subscriptions 44714b32dec065550964e6143e02500a9a9e22789c6e6e32aff0e6f81401d5db network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-subscriptions 44714b32dec065550964e6143e02500a9a9e22789c6e6e32aff0e6f81401d5db network default NAD default NAD key ] I0505 19:57:21.822764 3666 cni.go:473] [tsf/test-tsf-subscriptions 44714b32dec065550964e6143e02500a9a9e22789c6e6e32aff0e6f81401d5db network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-subscriptions 44714b32dec065550964e6143e02500a9a9e22789c6e6e32aff0e6f81401d5db network default NAD default NAD key default], result "{}", err I0505 19:57:21.879407 3666 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:57:21.879435 3666 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0505 19:57:21.880810 3666 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.24 I0505 19:57:21.880842 3666 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:57:21.911675 3666 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:57:21.911695 3666 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:57:21.920413 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0505 19:57:21.920440 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions W0505 19:57:23.291120 3666 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0505 19:57:23.291141 3666 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0505 19:57:23.291147 3666 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0505 19:57:23.291153 3666 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0505 19:57:23.396756 3666 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0505 19:57:23.396774 3666 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0505 19:57:23.396778 3666 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0505 19:57:23.923237 3666 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-2-97.ec2.internal I0505 19:57:23.923345 3666 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0505 19:57:23.950915 3666 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 27.609636ms I0505 19:57:23.950940 3666 base_network_controller_pods.go:951] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 27.636595ms I0505 19:57:23.952148 3666 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 28.929801ms, libovsdb time 844.275µs I0505 19:57:24.293664 3666 cni.go:452] [tsf/test-tsf-infrastructure e20826e3271f5bdd7c28383df9e21b6e65c8e07a28366c6250f86dbbb3c33628 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-infrastructure e20826e3271f5bdd7c28383df9e21b6e65c8e07a28366c6250f86dbbb3c33628 network default NAD default NAD key ] I0505 19:57:24.303867 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: e20826e3271f5bd, network: default, NAD default, SandboxID: "e20826e3271f5bdd7c28383df9e21b6e65c8e07a28366c6250f86dbbb3c33628", PCI device ID: , UID: "09cb1ae4-a926-4faf-8101-f638b70319c1", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0505 19:57:24.352607 3666 cni.go:473] [tsf/test-tsf-infrastructure e20826e3271f5bdd7c28383df9e21b6e65c8e07a28366c6250f86dbbb3c33628 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-infrastructure e20826e3271f5bdd7c28383df9e21b6e65c8e07a28366c6250f86dbbb3c33628 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:5f:39:4d:e5:0e\",\"name\":\"e20826e3271f5bd\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2ca879ef-716c-4e50-b0c4-763833f20fa5\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:57:26.907600 3666 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-6p2rd] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-6p2rd for pod on switch ip-10-0-2-97.ec2.internal I0505 19:57:26.907745 3666 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-6p2rd I0505 19:57:26.922858 3666 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-6p2rd] pod update took 15.177681ms I0505 19:57:26.922886 3666 base_network_controller_pods.go:951] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-6p2rd] addLogicalPort annotation time took 15.214247ms I0505 19:57:26.924413 3666 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-6p2rd] addLogicalPort took 16.835269ms, libovsdb time 1.153022ms I0505 19:57:26.941327 3666 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-gcchx] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-gcchx for pod on switch ip-10-0-2-97.ec2.internal I0505 19:57:26.941418 3666 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-gcchx I0505 19:57:26.959578 3666 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-gcchx] pod update took 18.191227ms I0505 19:57:26.959647 3666 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-gcchx] addLogicalPort annotation time took 18.260084ms I0505 19:57:26.960891 3666 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-gcchx] addLogicalPort took 19.57707ms, libovsdb time 870.139µs W0505 19:57:31.745131 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 19:57:31.745156 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0505 19:57:31.745162 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0505 19:57:35.967783 3666 cni.go:452] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-6p2rd 4b37120d915ac4afe196042ccd80a6a7ad8e1044c21cf5f979a43bf2d64e89c8 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-6p2rd 4b37120d915ac4afe196042ccd80a6a7ad8e1044c21cf5f979a43bf2d64e89c8 network default NAD default NAD key ] I0505 19:57:35.977226 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-6p2rd, hostIfaceName: 4b37120d915ac4a, network: default, NAD default, SandboxID: "4b37120d915ac4afe196042ccd80a6a7ad8e1044c21cf5f979a43bf2d64e89c8", PCI device ID: , UID: "c87396bf-caae-4a77-b26f-f3fdc63a88b3", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0505 19:57:36.003971 3666 cni.go:452] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-gcchx a22beab9590eb4085ed98cc2f21283723be0e73c916fcb46b18477dadbcb7a33 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-gcchx a22beab9590eb4085ed98cc2f21283723be0e73c916fcb46b18477dadbcb7a33 network default NAD default NAD key ] I0505 19:57:36.034561 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-gcchx, hostIfaceName: a22beab9590eb40, network: default, NAD default, SandboxID: "a22beab9590eb4085ed98cc2f21283723be0e73c916fcb46b18477dadbcb7a33", PCI device ID: , UID: "737f5cea-38f3-4dd2-81d3-6df906c4bf59", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0505 19:57:36.040989 3666 cni.go:473] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-6p2rd 4b37120d915ac4afe196042ccd80a6a7ad8e1044c21cf5f979a43bf2d64e89c8 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-6p2rd 4b37120d915ac4afe196042ccd80a6a7ad8e1044c21cf5f979a43bf2d64e89c8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:5e:b6:a9:ff:ec\",\"name\":\"4b37120d915ac4a\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8104861d-7602-4bda-a871-f5b0f3095bc7\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:57:36.082207 3666 cni.go:473] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-gcchx a22beab9590eb4085ed98cc2f21283723be0e73c916fcb46b18477dadbcb7a33 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-gcchx a22beab9590eb4085ed98cc2f21283723be0e73c916fcb46b18477dadbcb7a33 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:45:a0:ef:95:c4\",\"name\":\"a22beab9590eb40\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/89efef28-549c-46d6-841c-e4ecbd63bfd1\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:57:36.842921 3666 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0505 19:57:36.874597 3666 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0505 19:57:41.825489 3666 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-l9ckw] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-l9ckw for pod on switch ip-10-0-2-97.ec2.internal I0505 19:57:41.825597 3666 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-l9ckw I0505 19:57:41.849230 3666 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-l9ckw] pod update took 23.657627ms I0505 19:57:41.849251 3666 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-l9ckw] addLogicalPort annotation time took 23.683299ms I0505 19:57:41.850381 3666 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-l9ckw] addLogicalPort took 24.913486ms, libovsdb time 803.956µs I0505 19:57:42.188479 3666 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-l9ckw 20c0f0283f1d6775ae7c91d6a5a308b065364527d02dd1a1be1d1972e004df89 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-l9ckw 20c0f0283f1d6775ae7c91d6a5a308b065364527d02dd1a1be1d1972e004df89 network default NAD default NAD key ] I0505 19:57:42.197534 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-l9ckw, hostIfaceName: 20c0f0283f1d677, network: default, NAD default, SandboxID: "20c0f0283f1d6775ae7c91d6a5a308b065364527d02dd1a1be1d1972e004df89", PCI device ID: , UID: "970b4bc3-873a-45e9-9e5b-5276a165f610", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0505 19:57:42.247773 3666 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-l9ckw 20c0f0283f1d6775ae7c91d6a5a308b065364527d02dd1a1be1d1972e004df89 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-l9ckw 20c0f0283f1d6775ae7c91d6a5a308b065364527d02dd1a1be1d1972e004df89 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:83:59:42:3d:3c\",\"name\":\"20c0f0283f1d677\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/817c49cc-18b1-49a7-82c1-c3a0decc1ea8\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:57:45.861546 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0505 19:57:56.870078 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0505 19:57:57.828751 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0505 19:57:57.868955 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0505 19:58:00.920689 3666 cni.go:452] [tsf/test-tsf-infrastructure e20826e3271f5bdd7c28383df9e21b6e65c8e07a28366c6250f86dbbb3c33628 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-infrastructure e20826e3271f5bdd7c28383df9e21b6e65c8e07a28366c6250f86dbbb3c33628 network default NAD default NAD key ] I0505 19:58:00.998762 3666 cni.go:473] [tsf/test-tsf-infrastructure e20826e3271f5bdd7c28383df9e21b6e65c8e07a28366c6250f86dbbb3c33628 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-infrastructure e20826e3271f5bdd7c28383df9e21b6e65c8e07a28366c6250f86dbbb3c33628 network default NAD default NAD key default], result "{}", err I0505 19:58:01.054125 3666 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:58:01.054158 3666 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0505 19:58:01.055665 3666 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.128.0.26 I0505 19:58:01.055694 3666 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:58:01.079828 3666 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:58:01.079852 3666 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:58:01.090989 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0505 19:58:01.091009 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0505 19:58:02.397791 3666 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:02.397953 3666 kube.go:256] Updating pod tsf/test-tsf-iam I0505 19:58:02.414651 3666 pod.go:62] [tsf/test-tsf-iam] pod update took 16.735334ms I0505 19:58:02.414672 3666 base_network_controller_pods.go:951] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 16.763089ms I0505 19:58:02.416199 3666 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 18.435467ms, libovsdb time 1.179291ms I0505 19:58:02.754460 3666 cni.go:452] [tsf/test-tsf-iam 5f17cd86ca043272a7d1d970cbd37d9eb828fa915d5cd5fcb78d28fc57d7f4d6 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-iam 5f17cd86ca043272a7d1d970cbd37d9eb828fa915d5cd5fcb78d28fc57d7f4d6 network default NAD default NAD key ] I0505 19:58:02.763738 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 5f17cd86ca04327, network: default, NAD default, SandboxID: "5f17cd86ca043272a7d1d970cbd37d9eb828fa915d5cd5fcb78d28fc57d7f4d6", PCI device ID: , UID: "ff83bb22-f87b-401e-9ffc-de78cd9b4dcd", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0505 19:58:02.812425 3666 cni.go:473] [tsf/test-tsf-iam 5f17cd86ca043272a7d1d970cbd37d9eb828fa915d5cd5fcb78d28fc57d7f4d6 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-iam 5f17cd86ca043272a7d1d970cbd37d9eb828fa915d5cd5fcb78d28fc57d7f4d6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:52:fd:8a:ea:d3\",\"name\":\"5f17cd86ca04327\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b5172f6b-4d0b-466d-8390-91d2e698d2ed\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:58:03.050822 3666 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0505 19:58:03.156685 3666 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0505 19:58:03.157317 3666 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0505 19:58:03.157336 3666 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0505 19:58:03.159376 3666 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0505 19:58:06.147468 3666 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:06.147601 3666 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0505 19:58:06.162286 3666 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 14.71736ms I0505 19:58:06.162309 3666 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 14.748265ms I0505 19:58:06.163557 3666 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 16.107095ms, libovsdb time 861.119µs W0505 19:58:09.544806 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0505 19:58:09.544825 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0505 19:58:09.548971 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0505 19:58:09.548986 3666 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0505 19:58:15.510039 3666 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 ca5b26a775fdf02cfd0de41e85808e1768ba2465c2296372d44038e4ef586c16 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 ca5b26a775fdf02cfd0de41e85808e1768ba2465c2296372d44038e4ef586c16 network default NAD default NAD key ] I0505 19:58:15.519549 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: ca5b26a775fdf02, network: default, NAD default, SandboxID: "ca5b26a775fdf02cfd0de41e85808e1768ba2465c2296372d44038e4ef586c16", PCI device ID: , UID: "120eb029-fd34-4ce4-b60c-791467e79f76", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0505 19:58:15.567221 3666 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 ca5b26a775fdf02cfd0de41e85808e1768ba2465c2296372d44038e4ef586c16 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 ca5b26a775fdf02cfd0de41e85808e1768ba2465c2296372d44038e4ef586c16 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:40:8f:f1:92:66\",\"name\":\"ca5b26a775fdf02\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b6fae278-53c2-4093-a6c6-a4f71940e543\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:58:21.081451 3666 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0505 19:58:22.513764 3666 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-fq2xn] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-585694b54-fq2xn for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:22.513865 3666 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-585694b54-fq2xn I0505 19:58:22.529635 3666 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-fq2xn] pod update took 15.793603ms I0505 19:58:22.529661 3666 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-fq2xn] addLogicalPort annotation time took 15.825492ms I0505 19:58:22.531084 3666 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-fq2xn] addLogicalPort took 17.335927ms, libovsdb time 978.37µs I0505 19:58:22.632381 3666 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-h7tkd] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-h7tkd for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:22.632534 3666 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-h7tkd I0505 19:58:22.653069 3666 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-h7tkd] pod update took 20.582919ms I0505 19:58:22.653089 3666 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-h7tkd] addLogicalPort annotation time took 20.608552ms I0505 19:58:22.654261 3666 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-h7tkd] addLogicalPort took 21.902826ms, libovsdb time 792.301µs I0505 19:58:22.873863 3666 cni.go:452] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-fq2xn 793cd4861203e497da527d6e997957cceb863490be70c6d389103ca868b74f9f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-fq2xn 793cd4861203e497da527d6e997957cceb863490be70c6d389103ca868b74f9f network default NAD default NAD key ] I0505 19:58:22.883440 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-585694b54-fq2xn, hostIfaceName: 793cd4861203e49, network: default, NAD default, SandboxID: "793cd4861203e497da527d6e997957cceb863490be70c6d389103ca868b74f9f", PCI device ID: , UID: "f12d0e70-2b0e-4646-8f64-3b2f1834068b", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0505 19:58:22.933905 3666 cni.go:473] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-fq2xn 793cd4861203e497da527d6e997957cceb863490be70c6d389103ca868b74f9f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-fq2xn 793cd4861203e497da527d6e997957cceb863490be70c6d389103ca868b74f9f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:8a:33:bb:8f:78\",\"name\":\"793cd4861203e49\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a704d1ba-4969-46b3-b65a-aae707df415b\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:58:22.991585 3666 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-h7tkd f8474e9934d503a3cd40128a4c8b7bce7806655ec30853c9a7825b8e33c36d50 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-h7tkd f8474e9934d503a3cd40128a4c8b7bce7806655ec30853c9a7825b8e33c36d50 network default NAD default NAD key ] I0505 19:58:23.002288 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-h7tkd, hostIfaceName: f8474e9934d503a, network: default, NAD default, SandboxID: "f8474e9934d503a3cd40128a4c8b7bce7806655ec30853c9a7825b8e33c36d50", PCI device ID: , UID: "4e9fd1f2-721c-4369-a18a-de85dc84db36", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0505 19:58:23.053804 3666 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-h7tkd f8474e9934d503a3cd40128a4c8b7bce7806655ec30853c9a7825b8e33c36d50 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-h7tkd f8474e9934d503a3cd40128a4c8b7bce7806655ec30853c9a7825b8e33c36d50 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:91:d3:4f:33:62\",\"name\":\"f8474e9934d503a\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/997562e9-7a65-4938-a56f-c942d3fbc79e\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:58:26.423853 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 W0505 19:58:27.958881 3666 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0505 19:58:27.958899 3666 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0505 19:58:27.958904 3666 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0505 19:58:28.446211 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0505 19:58:32.327606 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=97 I0505 19:58:39.121868 3666 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-z2298] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-z2298 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:39.121963 3666 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-z2298 I0505 19:58:39.137014 3666 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-z2298] pod update took 15.075777ms I0505 19:58:39.137035 3666 base_network_controller_pods.go:951] [default/openshift-pipelines/tkn-cli-serve-784b566968-z2298] addLogicalPort annotation time took 15.102228ms I0505 19:58:39.138229 3666 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-z2298] addLogicalPort took 16.382765ms, libovsdb time 825.592µs I0505 19:58:39.479509 3666 cni.go:452] [openshift-pipelines/tkn-cli-serve-784b566968-z2298 49062096acf9b59402b94b6b0f5993393ce969b514d151409b9cb5d35f5b34df network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-z2298 49062096acf9b59402b94b6b0f5993393ce969b514d151409b9cb5d35f5b34df network default NAD default NAD key ] I0505 19:58:39.488741 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-z2298, hostIfaceName: 49062096acf9b59, network: default, NAD default, SandboxID: "49062096acf9b59402b94b6b0f5993393ce969b514d151409b9cb5d35f5b34df", PCI device ID: , UID: "389395a3-6c37-4a0d-bea9-c1eadbfbe979", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0505 19:58:39.538657 3666 cni.go:473] [openshift-pipelines/tkn-cli-serve-784b566968-z2298 49062096acf9b59402b94b6b0f5993393ce969b514d151409b9cb5d35f5b34df network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-z2298 49062096acf9b59402b94b6b0f5993393ce969b514d151409b9cb5d35f5b34df network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:83:bd:11:c9:05\",\"name\":\"49062096acf9b59\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4fe8f56f-2f4e-4dcc-85a7-0aa7404c578a\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:58:49.445853 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0505 19:58:54.193934 3666 cni.go:452] [tsf/test-tsf-iam 5f17cd86ca043272a7d1d970cbd37d9eb828fa915d5cd5fcb78d28fc57d7f4d6 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-iam 5f17cd86ca043272a7d1d970cbd37d9eb828fa915d5cd5fcb78d28fc57d7f4d6 network default NAD default NAD key ] I0505 19:58:54.277900 3666 cni.go:473] [tsf/test-tsf-iam 5f17cd86ca043272a7d1d970cbd37d9eb828fa915d5cd5fcb78d28fc57d7f4d6 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-iam 5f17cd86ca043272a7d1d970cbd37d9eb828fa915d5cd5fcb78d28fc57d7f4d6 network default NAD default NAD key default], result "{}", err I0505 19:58:54.441665 3666 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:58:54.441703 3666 pods.go:176] Deleting pod: tsf/test-tsf-iam I0505 19:58:54.443017 3666 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.30 I0505 19:58:54.443049 3666 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:58:54.496756 3666 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:58:54.496777 3666 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:58:54.505789 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0505 19:58:54.505847 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0505 19:58:55.516002 3666 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:55.516138 3666 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0505 19:58:55.536939 3666 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 20.825086ms I0505 19:58:55.536959 3666 base_network_controller_pods.go:951] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 20.849126ms I0505 19:58:55.538175 3666 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 22.187096ms, libovsdb time 874.554µs I0505 19:58:55.963053 3666 cni.go:452] [tsf-tas/test-tsf-tas f42e2465dbf996d47fc9e8bd31712c384c44c5f19a6d908f97583c140d7b5b7e network default NAD default NAD key ] ADD starting CNI request [tsf-tas/test-tsf-tas f42e2465dbf996d47fc9e8bd31712c384c44c5f19a6d908f97583c140d7b5b7e network default NAD default NAD key ] I0505 19:58:55.972224 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: f42e2465dbf996d, network: default, NAD default, SandboxID: "f42e2465dbf996d47fc9e8bd31712c384c44c5f19a6d908f97583c140d7b5b7e", PCI device ID: , UID: "b81ef641-85f8-41cb-b3a1-2d5c29c8a083", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0505 19:58:56.022117 3666 cni.go:473] [tsf-tas/test-tsf-tas f42e2465dbf996d47fc9e8bd31712c384c44c5f19a6d908f97583c140d7b5b7e network default NAD default NAD key default] ADD finished CNI request [tsf-tas/test-tsf-tas f42e2465dbf996d47fc9e8bd31712c384c44c5f19a6d908f97583c140d7b5b7e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:94:35:d1:19:97\",\"name\":\"f42e2465dbf996d\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fff18ea3-7444-4b4e-967b-262f94fcd35b\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:58:57.364709 3666 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-879784cfb-xcwtc] creating logical port tsf-tas_fulcio-server-879784cfb-xcwtc for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:57.364833 3666 kube.go:256] Updating pod tsf-tas/fulcio-server-879784cfb-xcwtc I0505 19:58:57.383919 3666 pod.go:62] [tsf-tas/fulcio-server-879784cfb-xcwtc] pod update took 19.111806ms I0505 19:58:57.383979 3666 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-879784cfb-xcwtc] addLogicalPort annotation time took 19.14816ms I0505 19:58:57.385418 3666 pods.go:271] [tsf-tas/fulcio-server-879784cfb-xcwtc] addLogicalPort took 20.727485ms, libovsdb time 947.79µs I0505 19:58:57.768081 3666 cni.go:452] [tsf-tas/fulcio-server-879784cfb-xcwtc ca493eeb4508394f056330d8326af27aeec3ec67be678472a3a0c665faedc0d5 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-879784cfb-xcwtc ca493eeb4508394f056330d8326af27aeec3ec67be678472a3a0c665faedc0d5 network default NAD default NAD key ] I0505 19:58:57.777215 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-879784cfb-xcwtc, hostIfaceName: ca493eeb4508394, network: default, NAD default, SandboxID: "ca493eeb4508394f056330d8326af27aeec3ec67be678472a3a0c665faedc0d5", PCI device ID: , UID: "77b164a8-be49-462e-bc2a-b157a8357b28", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0505 19:58:57.829784 3666 cni.go:473] [tsf-tas/fulcio-server-879784cfb-xcwtc ca493eeb4508394f056330d8326af27aeec3ec67be678472a3a0c665faedc0d5 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-879784cfb-xcwtc ca493eeb4508394f056330d8326af27aeec3ec67be678472a3a0c665faedc0d5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:40:89:a5:51:e3\",\"name\":\"ca493eeb4508394\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f06b8aa-86a5-4a62-ab28-878816dd6bf0\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:58:58.014752 3666 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bsd78] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-bsd78 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:58.014823 3666 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-6xtqg-xl244] creating logical port tsf-tas_rekor-createtree-job-6xtqg-xl244 for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:58.014877 3666 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bsd78 I0505 19:58:58.014905 3666 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-6xtqg-xl244 I0505 19:58:58.036864 3666 pod.go:62] [tsf-tas/rekor-createtree-job-6xtqg-xl244] pod update took 21.98066ms I0505 19:58:58.036886 3666 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-6xtqg-xl244] addLogicalPort annotation time took 22.007377ms I0505 19:58:58.037810 3666 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bsd78] pod update took 22.956186ms I0505 19:58:58.037829 3666 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bsd78] addLogicalPort annotation time took 22.980458ms I0505 19:58:58.038111 3666 pods.go:271] [tsf-tas/rekor-createtree-job-6xtqg-xl244] addLogicalPort took 23.303648ms, libovsdb time 874.026µs I0505 19:58:58.039064 3666 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bsd78] addLogicalPort took 24.333951ms, libovsdb time 722.003µs I0505 19:58:58.256326 3666 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-75cd874f-lw46c] creating logical port tsf-tas_trillian-logsigner-75cd874f-lw46c for pod on switch ip-10-0-2-97.ec2.internal I0505 19:58:58.256470 3666 kube.go:256] Updating pod tsf-tas/trillian-logsigner-75cd874f-lw46c I0505 19:58:58.272653 3666 pod.go:62] [tsf-tas/trillian-logsigner-75cd874f-lw46c] pod update took 16.218678ms I0505 19:58:58.272676 3666 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logsigner-75cd874f-lw46c] addLogicalPort annotation time took 16.247659ms I0505 19:58:58.273862 3666 pods.go:271] [tsf-tas/trillian-logsigner-75cd874f-lw46c] addLogicalPort took 17.552335ms, libovsdb time 830.37µs I0505 19:58:58.431073 3666 cni.go:452] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bsd78 43bda2330fbc0a972d6dead9810ee0845aee2a885cbadfff806a8ffdca26ce0f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bsd78 43bda2330fbc0a972d6dead9810ee0845aee2a885cbadfff806a8ffdca26ce0f network default NAD default NAD key ] I0505 19:58:58.440968 3666 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-bsd78, hostIfaceName: 43bda2330fbc0a9, network: default, NAD default, SandboxID: "43bda2330fbc0a972d6dead9810ee0845aee2a885cbadfff806a8ffdca26ce0f", PCI device ID: , UID: "af6989f4-c5d9-4a54-ab4f-c370dc6f3de3", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0505 19:58:58.470859 3666 cni.go:452] [tsf-tas/rekor-createtree-job-6xtqg-xl244 d6f8172da60f4a9ea203ace29c271fb14c6de2c73c8d22d8c810f9de03a38215 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-6xtqg-xl244 d6f8172da60f4a9ea203ace29c271fb14c6de2c73c8d22d8c810f9de03a38215 network default NAD default NAD key ] I0505 19:58:58.483172 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-6xtqg-xl244, hostIfaceName: d6f8172da60f4a9, network: default, NAD default, SandboxID: "d6f8172da60f4a9ea203ace29c271fb14c6de2c73c8d22d8c810f9de03a38215", PCI device ID: , UID: "ccda7687-e635-4581-9d87-49ddddcd3e77", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0505 19:58:58.498460 3666 cni.go:473] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bsd78 43bda2330fbc0a972d6dead9810ee0845aee2a885cbadfff806a8ffdca26ce0f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-bsd78 43bda2330fbc0a972d6dead9810ee0845aee2a885cbadfff806a8ffdca26ce0f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:d2:dc:65:d8:a0\",\"name\":\"43bda2330fbc0a9\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a2d52459-4d60-4614-9f48-3231cb9e1721\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:58:58.539945 3666 cni.go:473] [tsf-tas/rekor-createtree-job-6xtqg-xl244 d6f8172da60f4a9ea203ace29c271fb14c6de2c73c8d22d8c810f9de03a38215 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-6xtqg-xl244 d6f8172da60f4a9ea203ace29c271fb14c6de2c73c8d22d8c810f9de03a38215 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:dc:a3:94:61:67\",\"name\":\"d6f8172da60f4a9\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b9920804-d761-49af-bdaf-ce7fa3f9694b\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:59:00.107753 3666 cni.go:452] [tsf-tas/trillian-logsigner-75cd874f-lw46c e5f17fc48451404786f559956aef78fde9d2ad2f5fda97415deb07e5f429d93c network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logsigner-75cd874f-lw46c e5f17fc48451404786f559956aef78fde9d2ad2f5fda97415deb07e5f429d93c network default NAD default NAD key ] I0505 19:59:00.119243 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-75cd874f-lw46c, hostIfaceName: e5f17fc48451404, network: default, NAD default, SandboxID: "e5f17fc48451404786f559956aef78fde9d2ad2f5fda97415deb07e5f429d93c", PCI device ID: , UID: "91535b47-8536-4bac-81b5-a58dd2756058", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0505 19:59:00.178936 3666 cni.go:473] [tsf-tas/trillian-logsigner-75cd874f-lw46c e5f17fc48451404786f559956aef78fde9d2ad2f5fda97415deb07e5f429d93c network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logsigner-75cd874f-lw46c e5f17fc48451404786f559956aef78fde9d2ad2f5fda97415deb07e5f429d93c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:6c:e9:fe:d9:ac\",\"name\":\"e5f17fc48451404\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9ff9a360-def3-4d46-99e8-7ffe619aebd1\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 19:59:01.249028 3666 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0505 19:59:01.249048 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0505 19:59:01.274139 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0505 19:59:01.274158 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server W0505 19:59:02.257135 3666 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0505 19:59:02.257188 3666 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0505 19:59:02.291086 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0505 19:59:02.291112 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0505 19:59:02.998960 3666 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-vsk5k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:59:02.999546 3666 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-vsk5k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:59:03.891356 3666 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-vsk5k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:59:03.891382 3666 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-vsk5k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0505 19:59:07.286051 3666 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0505 19:59:10.202201 3666 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0505 19:59:10.273258 3666 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0505 19:59:13.210998 3666 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0505 19:59:13.357602 3666 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-hcq4q-jvnff of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:59:13.358153 3666 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-hcq4q-jvnff of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:59:14.382232 3666 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-hcq4q-jvnff of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:59:14.382256 3666 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-hcq4q-jvnff of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:59:19.429001 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 W0505 19:59:41.413510 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0505 19:59:45.473119 3666 cni.go:452] [tsf-tas/test-tsf-tas f42e2465dbf996d47fc9e8bd31712c384c44c5f19a6d908f97583c140d7b5b7e network default NAD default NAD key ] DEL starting CNI request [tsf-tas/test-tsf-tas f42e2465dbf996d47fc9e8bd31712c384c44c5f19a6d908f97583c140d7b5b7e network default NAD default NAD key ] I0505 19:59:45.558057 3666 cni.go:473] [tsf-tas/test-tsf-tas f42e2465dbf996d47fc9e8bd31712c384c44c5f19a6d908f97583c140d7b5b7e network default NAD default NAD key default] DEL finished CNI request [tsf-tas/test-tsf-tas f42e2465dbf996d47fc9e8bd31712c384c44c5f19a6d908f97583c140d7b5b7e network default NAD default NAD key default], result "{}", err I0505 19:59:45.613904 3666 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:59:45.613936 3666 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0505 19:59:45.615461 3666 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.128.0.35 I0505 19:59:45.615493 3666 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:59:45.640024 3666 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:59:45.640051 3666 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:59:45.644030 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0505 19:59:45.644058 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0505 19:59:46.493577 3666 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-fx4jr] creating logical port tsf_patch-tekton-config-fx4jr for pod on switch ip-10-0-2-97.ec2.internal I0505 19:59:46.493675 3666 kube.go:256] Updating pod tsf/patch-tekton-config-fx4jr I0505 19:59:46.509306 3666 pod.go:62] [tsf/patch-tekton-config-fx4jr] pod update took 15.65681ms I0505 19:59:46.509327 3666 base_network_controller_pods.go:951] [default/tsf/patch-tekton-config-fx4jr] addLogicalPort annotation time took 15.686387ms I0505 19:59:46.510673 3666 pods.go:271] [tsf/patch-tekton-config-fx4jr] addLogicalPort took 17.116325ms, libovsdb time 920.831µs I0505 19:59:46.853175 3666 cni.go:452] [tsf/patch-tekton-config-fx4jr 7ff57f84ffdd48f5431516eacc820088f834af54b997b8de04814820315cd370 network default NAD default NAD key ] ADD starting CNI request [tsf/patch-tekton-config-fx4jr 7ff57f84ffdd48f5431516eacc820088f834af54b997b8de04814820315cd370 network default NAD default NAD key ] I0505 19:59:46.862498 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-fx4jr, hostIfaceName: 7ff57f84ffdd48f, network: default, NAD default, SandboxID: "7ff57f84ffdd48f5431516eacc820088f834af54b997b8de04814820315cd370", PCI device ID: , UID: "cb47ae27-0ca5-45ef-b890-e7400041b7d0", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0505 19:59:46.913809 3666 cni.go:473] [tsf/patch-tekton-config-fx4jr 7ff57f84ffdd48f5431516eacc820088f834af54b997b8de04814820315cd370 network default NAD default NAD key default] ADD finished CNI request [tsf/patch-tekton-config-fx4jr 7ff57f84ffdd48f5431516eacc820088f834af54b997b8de04814820315cd370 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:69:58:48:54:7e\",\"name\":\"7ff57f84ffdd48f\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/63842cc4-08d2-44a6-8e76-a45609ab4bfc\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:59:54.528989 3666 cni.go:452] [tsf/patch-tekton-config-fx4jr 7ff57f84ffdd48f5431516eacc820088f834af54b997b8de04814820315cd370 network default NAD default NAD key ] DEL starting CNI request [tsf/patch-tekton-config-fx4jr 7ff57f84ffdd48f5431516eacc820088f834af54b997b8de04814820315cd370 network default NAD default NAD key ] I0505 19:59:54.614298 3666 cni.go:473] [tsf/patch-tekton-config-fx4jr 7ff57f84ffdd48f5431516eacc820088f834af54b997b8de04814820315cd370 network default NAD default NAD key default] DEL finished CNI request [tsf/patch-tekton-config-fx4jr 7ff57f84ffdd48f5431516eacc820088f834af54b997b8de04814820315cd370 network default NAD default NAD key default], result "{}", err I0505 19:59:54.667723 3666 obj_retry.go:498] Detected object tsf/patch-tekton-config-fx4jr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:59:54.667760 3666 pods.go:176] Deleting pod: tsf/patch-tekton-config-fx4jr I0505 19:59:54.669002 3666 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-fx4jr, ips: 10.128.0.40 I0505 19:59:54.669026 3666 obj_retry.go:498] Detected object tsf/patch-tekton-config-fx4jr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 19:59:55.694105 3666 obj_retry.go:492] Detected object tsf/patch-tekton-config-fx4jr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:59:55.694131 3666 obj_retry.go:492] Detected object tsf/patch-tekton-config-fx4jr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 19:59:55.824251 3666 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-wq45v] creating logical port tsf_tsf-tekton-configuration-wq45v for pod on switch ip-10-0-2-97.ec2.internal I0505 19:59:55.824348 3666 kube.go:256] Updating pod tsf/tsf-tekton-configuration-wq45v I0505 19:59:55.839640 3666 pod.go:62] [tsf/tsf-tekton-configuration-wq45v] pod update took 15.318882ms I0505 19:59:55.839666 3666 base_network_controller_pods.go:951] [default/tsf/tsf-tekton-configuration-wq45v] addLogicalPort annotation time took 15.347181ms I0505 19:59:55.840928 3666 pods.go:271] [tsf/tsf-tekton-configuration-wq45v] addLogicalPort took 16.69109ms, libovsdb time 874.021µs I0505 19:59:56.185316 3666 cni.go:452] [tsf/tsf-tekton-configuration-wq45v 664a3d7d7d28846fa5f436b4cc71627e60bbac88b11b21de6e2c0f4bd57c9fbd network default NAD default NAD key ] ADD starting CNI request [tsf/tsf-tekton-configuration-wq45v 664a3d7d7d28846fa5f436b4cc71627e60bbac88b11b21de6e2c0f4bd57c9fbd network default NAD default NAD key ] I0505 19:59:56.194495 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-wq45v, hostIfaceName: 664a3d7d7d28846, network: default, NAD default, SandboxID: "664a3d7d7d28846fa5f436b4cc71627e60bbac88b11b21de6e2c0f4bd57c9fbd", PCI device ID: , UID: "1444451f-b62e-4dc0-85c9-c536beb714a0", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0505 19:59:56.245871 3666 cni.go:473] [tsf/tsf-tekton-configuration-wq45v 664a3d7d7d28846fa5f436b4cc71627e60bbac88b11b21de6e2c0f4bd57c9fbd network default NAD default NAD key default] ADD finished CNI request [tsf/tsf-tekton-configuration-wq45v 664a3d7d7d28846fa5f436b4cc71627e60bbac88b11b21de6e2c0f4bd57c9fbd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:86:08:0b:03:e6\",\"name\":\"664a3d7d7d28846\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/024f4214-c2c8-4d82-bf17-b2b75eb4e1e8\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 19:59:59.605037 3666 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-n7qmb-4b4xl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 19:59:59.605666 3666 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-n7qmb-4b4xl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:00.479630 3666 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-n7qmb-4b4xl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:00.479662 3666 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-n7qmb-4b4xl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:00.569196 3666 cni.go:452] [tsf-tas/rekor-createtree-job-6xtqg-xl244 d6f8172da60f4a9ea203ace29c271fb14c6de2c73c8d22d8c810f9de03a38215 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-6xtqg-xl244 d6f8172da60f4a9ea203ace29c271fb14c6de2c73c8d22d8c810f9de03a38215 network default NAD default NAD key ] I0505 20:00:00.657064 3666 cni.go:473] [tsf-tas/rekor-createtree-job-6xtqg-xl244 d6f8172da60f4a9ea203ace29c271fb14c6de2c73c8d22d8c810f9de03a38215 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-6xtqg-xl244 d6f8172da60f4a9ea203ace29c271fb14c6de2c73c8d22d8c810f9de03a38215 network default NAD default NAD key default], result "{}", err I0505 20:00:00.711562 3666 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-6xtqg-xl244 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:00.711592 3666 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-6xtqg-xl244 I0505 20:00:00.712937 3666 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-6xtqg-xl244, ips: 10.128.0.38 I0505 20:00:00.712962 3666 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-6xtqg-xl244 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:01.588096 3666 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-6xtqg-xl244 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:01.588120 3666 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-6xtqg-xl244 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:02.576541 3666 cni.go:452] [tsf/tsf-tekton-configuration-wq45v 664a3d7d7d28846fa5f436b4cc71627e60bbac88b11b21de6e2c0f4bd57c9fbd network default NAD default NAD key ] DEL starting CNI request [tsf/tsf-tekton-configuration-wq45v 664a3d7d7d28846fa5f436b4cc71627e60bbac88b11b21de6e2c0f4bd57c9fbd network default NAD default NAD key ] I0505 20:00:02.662967 3666 cni.go:473] [tsf/tsf-tekton-configuration-wq45v 664a3d7d7d28846fa5f436b4cc71627e60bbac88b11b21de6e2c0f4bd57c9fbd network default NAD default NAD key default] DEL finished CNI request [tsf/tsf-tekton-configuration-wq45v 664a3d7d7d28846fa5f436b4cc71627e60bbac88b11b21de6e2c0f4bd57c9fbd network default NAD default NAD key default], result "{}", err I0505 20:00:02.715544 3666 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-wq45v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:02.715578 3666 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-wq45v I0505 20:00:02.716952 3666 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-wq45v, ips: 10.128.0.41 I0505 20:00:02.716982 3666 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-wq45v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:03.740358 3666 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-wq45v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:03.740395 3666 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-wq45v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:03.789410 3666 obj_retry.go:492] Detected object tsf/patch-tekton-config-fx4jr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:03.789427 3666 obj_retry.go:492] Detected object tsf/patch-tekton-config-fx4jr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:03.798040 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-fx4jr I0505 20:00:03.798064 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-fx4jr I0505 20:00:04.347988 3666 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29633520-42fwf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:04.348648 3666 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29633520-42fwf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:04.625647 3666 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-2-97.ec2.internal I0505 20:00:04.625737 3666 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0505 20:00:04.646042 3666 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 20.33024ms I0505 20:00:04.646062 3666 base_network_controller_pods.go:951] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 20.35485ms I0505 20:00:04.647322 3666 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 21.692617ms, libovsdb time 924.358µs I0505 20:00:04.984271 3666 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 4c0c2951e7995e7928d4785c36789d255ace40f2b24acf6dbdc1b49cc81f255f network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 4c0c2951e7995e7928d4785c36789d255ace40f2b24acf6dbdc1b49cc81f255f network default NAD default NAD key ] I0505 20:00:04.993585 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 4c0c2951e7995e7, network: default, NAD default, SandboxID: "4c0c2951e7995e7928d4785c36789d255ace40f2b24acf6dbdc1b49cc81f255f", PCI device ID: , UID: "055bf01c-56cb-41d3-a00a-08d183927070", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0505 20:00:05.044103 3666 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 4c0c2951e7995e7928d4785c36789d255ace40f2b24acf6dbdc1b49cc81f255f network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 4c0c2951e7995e7928d4785c36789d255ace40f2b24acf6dbdc1b49cc81f255f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:a1:fb:f4:03:8d\",\"name\":\"4c0c2951e7995e7\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a2d446c8-86cd-4051-af3f-537e904ad52c\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 20:00:05.220506 3666 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29633520-42fwf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:05.220548 3666 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29633520-42fwf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0505 20:00:07.504916 3666 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0505 20:00:08.614153 3666 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 4c0c2951e7995e7928d4785c36789d255ace40f2b24acf6dbdc1b49cc81f255f network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 4c0c2951e7995e7928d4785c36789d255ace40f2b24acf6dbdc1b49cc81f255f network default NAD default NAD key ] I0505 20:00:08.696302 3666 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 4c0c2951e7995e7928d4785c36789d255ace40f2b24acf6dbdc1b49cc81f255f network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 4c0c2951e7995e7928d4785c36789d255ace40f2b24acf6dbdc1b49cc81f255f network default NAD default NAD key default], result "{}", err I0505 20:00:08.750737 3666 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:08.750778 3666 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0505 20:00:08.752118 3666 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.128.0.42 I0505 20:00:08.752148 3666 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:08.775167 3666 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:08.775193 3666 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:08.789724 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0505 20:00:08.789752 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0505 20:00:09.056007 3666 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-2-97.ec2.internal I0505 20:00:09.056118 3666 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0505 20:00:09.076121 3666 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 20.033526ms I0505 20:00:09.076142 3666 base_network_controller_pods.go:951] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 20.064255ms I0505 20:00:09.077468 3666 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 21.479707ms, libovsdb time 970.837µs I0505 20:00:09.416348 3666 cni.go:452] [tsf-tpa/test-tsf-tpa a06aeb1d3e2000f1c1a2e5739b71318aa01ae8300254b23a7bed4c434fcb168c network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/test-tsf-tpa a06aeb1d3e2000f1c1a2e5739b71318aa01ae8300254b23a7bed4c434fcb168c network default NAD default NAD key ] I0505 20:00:09.426128 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: a06aeb1d3e2000f, network: default, NAD default, SandboxID: "a06aeb1d3e2000f1c1a2e5739b71318aa01ae8300254b23a7bed4c434fcb168c", PCI device ID: , UID: "fa0e4dac-eee0-4904-abb1-101cd6e164de", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0505 20:00:09.478160 3666 cni.go:473] [tsf-tpa/test-tsf-tpa a06aeb1d3e2000f1c1a2e5739b71318aa01ae8300254b23a7bed4c434fcb168c network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/test-tsf-tpa a06aeb1d3e2000f1c1a2e5739b71318aa01ae8300254b23a7bed4c434fcb168c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:20:8a:1e:59:0d\",\"name\":\"a06aeb1d3e2000f\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0689e9b-b488-4fc0-ac51-241625b365cf\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 20:00:10.200221 3666 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-ngln5] creating logical port tsf-tpa_create-db-ngln5 for pod on switch ip-10-0-2-97.ec2.internal I0505 20:00:10.200375 3666 kube.go:256] Updating pod tsf-tpa/create-db-ngln5 I0505 20:00:10.217960 3666 pod.go:62] [tsf-tpa/create-db-ngln5] pod update took 17.642611ms I0505 20:00:10.217989 3666 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-ngln5] addLogicalPort annotation time took 17.675955ms I0505 20:00:10.219255 3666 pods.go:271] [tsf-tpa/create-db-ngln5] addLogicalPort took 19.05073ms, libovsdb time 899.216µs W0505 20:00:10.258492 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0505 20:00:10.567115 3666 cni.go:452] [tsf-tpa/create-db-ngln5 2b6ded8b5cc851e44d40e40d11863520247544664cec25104275ea34f449ca2f network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-ngln5 2b6ded8b5cc851e44d40e40d11863520247544664cec25104275ea34f449ca2f network default NAD default NAD key ] I0505 20:00:10.577753 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-ngln5, hostIfaceName: 2b6ded8b5cc851e, network: default, NAD default, SandboxID: "2b6ded8b5cc851e44d40e40d11863520247544664cec25104275ea34f449ca2f", PCI device ID: , UID: "fdfbea99-d219-44c0-8be5-2a3ba846fec7", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0505 20:00:10.629653 3666 cni.go:473] [tsf-tpa/create-db-ngln5 2b6ded8b5cc851e44d40e40d11863520247544664cec25104275ea34f449ca2f network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-ngln5 2b6ded8b5cc851e44d40e40d11863520247544664cec25104275ea34f449ca2f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:6f:7e:20:39:c1\",\"name\":\"2b6ded8b5cc851e\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fc0453f6-abfb-4d66-98cf-e4489b763f86\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 20:00:15.657781 3666 cni.go:452] [tsf-tpa/create-db-ngln5 2b6ded8b5cc851e44d40e40d11863520247544664cec25104275ea34f449ca2f network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-ngln5 2b6ded8b5cc851e44d40e40d11863520247544664cec25104275ea34f449ca2f network default NAD default NAD key ] I0505 20:00:15.745537 3666 cni.go:473] [tsf-tpa/create-db-ngln5 2b6ded8b5cc851e44d40e40d11863520247544664cec25104275ea34f449ca2f network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-ngln5 2b6ded8b5cc851e44d40e40d11863520247544664cec25104275ea34f449ca2f network default NAD default NAD key default], result "{}", err I0505 20:00:15.800041 3666 obj_retry.go:498] Detected object tsf-tpa/create-db-ngln5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:15.800075 3666 pods.go:176] Deleting pod: tsf-tpa/create-db-ngln5 I0505 20:00:15.801469 3666 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-ngln5, ips: 10.128.0.44 I0505 20:00:15.801495 3666 obj_retry.go:498] Detected object tsf-tpa/create-db-ngln5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:16.826867 3666 obj_retry.go:492] Detected object tsf-tpa/create-db-ngln5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:16.826893 3666 obj_retry.go:492] Detected object tsf-tpa/create-db-ngln5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0505 20:00:21.326714 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0505 20:00:21.585734 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0505 20:00:23.453495 3666 obj_retry.go:498] Detected object tsf-tpa/migrate-db-8mhms of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:23.454054 3666 obj_retry.go:498] Detected object tsf-tpa/migrate-db-8mhms of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:24.331206 3666 obj_retry.go:492] Detected object tsf-tpa/migrate-db-8mhms of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:24.331232 3666 obj_retry.go:492] Detected object tsf-tpa/migrate-db-8mhms of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:26.466688 3666 obj_retry.go:498] Detected object tsf-tpa/create-importers-wpq45 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:26.467213 3666 obj_retry.go:498] Detected object tsf-tpa/create-importers-wpq45 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:27.493892 3666 obj_retry.go:492] Detected object tsf-tpa/create-importers-wpq45 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:27.493916 3666 obj_retry.go:492] Detected object tsf-tpa/create-importers-wpq45 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:28.473503 3666 base_network_controller_pods.go:487] [default/tsf-tpa/server-65f5dcd55f-ffv5v] creating logical port tsf-tpa_server-65f5dcd55f-ffv5v for pod on switch ip-10-0-2-97.ec2.internal I0505 20:00:28.473620 3666 kube.go:256] Updating pod tsf-tpa/server-65f5dcd55f-ffv5v I0505 20:00:28.490146 3666 pod.go:62] [tsf-tpa/server-65f5dcd55f-ffv5v] pod update took 16.553118ms I0505 20:00:28.490169 3666 base_network_controller_pods.go:951] [default/tsf-tpa/server-65f5dcd55f-ffv5v] addLogicalPort annotation time took 16.581124ms I0505 20:00:28.491434 3666 pods.go:271] [tsf-tpa/server-65f5dcd55f-ffv5v] addLogicalPort took 17.946217ms, libovsdb time 855.057µs I0505 20:00:28.527500 3666 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-26r4c] creating logical port tsf-tpa_importer-86bcdf477b-26r4c for pod on switch ip-10-0-2-97.ec2.internal I0505 20:00:28.527610 3666 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-26r4c I0505 20:00:28.545152 3666 pod.go:62] [tsf-tpa/importer-86bcdf477b-26r4c] pod update took 17.566089ms I0505 20:00:28.545209 3666 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-26r4c] addLogicalPort annotation time took 17.59969ms I0505 20:00:28.546801 3666 pods.go:271] [tsf-tpa/importer-86bcdf477b-26r4c] addLogicalPort took 19.312915ms, libovsdb time 1.051794ms I0505 20:00:36.407177 3666 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0505 20:00:38.877072 3666 cni.go:452] [tsf-tpa/server-65f5dcd55f-ffv5v 2c1a5de5fcb1e0cddb9abc5b5e2b04b6c756a4850fb8ef6b4f9789cd336166a4 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-65f5dcd55f-ffv5v 2c1a5de5fcb1e0cddb9abc5b5e2b04b6c756a4850fb8ef6b4f9789cd336166a4 network default NAD default NAD key ] I0505 20:00:38.878947 3666 cni.go:452] [tsf-tpa/importer-86bcdf477b-26r4c 6d92a0f882f63a46274a201a51112b1c8ec3db72f395f3c8135a7fdca900c4af network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-26r4c 6d92a0f882f63a46274a201a51112b1c8ec3db72f395f3c8135a7fdca900c4af network default NAD default NAD key ] I0505 20:00:38.888756 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-26r4c, hostIfaceName: 6d92a0f882f63a4, network: default, NAD default, SandboxID: "6d92a0f882f63a46274a201a51112b1c8ec3db72f395f3c8135a7fdca900c4af", PCI device ID: , UID: "4dafd4ab-9949-45fb-9afb-a1e3eb87b4cd", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0505 20:00:38.888875 3666 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-65f5dcd55f-ffv5v, hostIfaceName: 2c1a5de5fcb1e0c, network: default, NAD default, SandboxID: "2c1a5de5fcb1e0cddb9abc5b5e2b04b6c756a4850fb8ef6b4f9789cd336166a4", PCI device ID: , UID: "9b6df644-9ee6-400c-8939-e3fe4507fa0d", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0505 20:00:38.939591 3666 cni.go:473] [tsf-tpa/importer-86bcdf477b-26r4c 6d92a0f882f63a46274a201a51112b1c8ec3db72f395f3c8135a7fdca900c4af network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-26r4c 6d92a0f882f63a46274a201a51112b1c8ec3db72f395f3c8135a7fdca900c4af network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:08:7a:92:a4:50\",\"name\":\"6d92a0f882f63a4\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2363aec9-adf5-4805-9cff-470a3239da1c\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0505 20:00:38.957888 3666 cni.go:473] [tsf-tpa/server-65f5dcd55f-ffv5v 2c1a5de5fcb1e0cddb9abc5b5e2b04b6c756a4850fb8ef6b4f9789cd336166a4 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-65f5dcd55f-ffv5v 2c1a5de5fcb1e0cddb9abc5b5e2b04b6c756a4850fb8ef6b4f9789cd336166a4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:c0:03:3a:e3:5e\",\"name\":\"2c1a5de5fcb1e0c\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/38402b9a-6b14-480b-9e8f-222742004750\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0505 20:00:39.809777 3666 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0505 20:00:50.801925 3666 util.go:959] Failed to build node endpoints for node ip-10-0-2-97.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0505 20:00:54.642236 3666 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-2hzjj-4djck of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:54.642797 3666 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-2hzjj-4djck of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:00:55.525207 3666 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-2hzjj-4djck of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:55.525229 3666 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-2hzjj-4djck of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:00:57.832877 3666 namespace.go:142] [openshift-pipelines] updating namespace I0505 20:01:02.901003 3666 cni.go:452] [tsf-tpa/test-tsf-tpa a06aeb1d3e2000f1c1a2e5739b71318aa01ae8300254b23a7bed4c434fcb168c network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/test-tsf-tpa a06aeb1d3e2000f1c1a2e5739b71318aa01ae8300254b23a7bed4c434fcb168c network default NAD default NAD key ] I0505 20:01:02.991557 3666 cni.go:473] [tsf-tpa/test-tsf-tpa a06aeb1d3e2000f1c1a2e5739b71318aa01ae8300254b23a7bed4c434fcb168c network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/test-tsf-tpa a06aeb1d3e2000f1c1a2e5739b71318aa01ae8300254b23a7bed4c434fcb168c network default NAD default NAD key default], result "{}", err I0505 20:01:03.050360 3666 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:01:03.050391 3666 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0505 20:01:03.051699 3666 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.128.0.43 I0505 20:01:03.051727 3666 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:01:03.096096 3666 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:01:03.096118 3666 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:01:03.106940 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0505 20:01:03.106970 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0505 20:01:04.019411 3666 namespace.go:98] [integration-service] adding namespace I0505 20:01:04.020293 3666 namespace.go:102] [integration-service] adding namespace took 862.12µs I0505 20:01:04.035869 3666 namespace.go:98] [build-service] adding namespace I0505 20:01:04.036031 3666 namespace.go:98] [image-controller] adding namespace I0505 20:01:04.036742 3666 namespace.go:102] [build-service] adding namespace took 856.844µs I0505 20:01:04.036776 3666 namespace.go:98] [default-tenant] adding namespace I0505 20:01:04.037311 3666 namespace.go:102] [default-tenant] adding namespace took 526.975µs I0505 20:01:04.037890 3666 namespace.go:102] [image-controller] adding namespace took 1.845743ms I0505 20:01:04.132556 3666 namespace.go:142] [integration-service] updating namespace I0505 20:01:04.184214 3666 namespace.go:142] [integration-service] updating namespace I0505 20:01:04.234644 3666 namespace.go:142] [build-service] updating namespace I0505 20:01:04.305260 3666 namespace.go:142] [build-service] updating namespace I0505 20:01:04.321201 3666 namespace.go:142] [default-tenant] updating namespace I0505 20:01:04.363763 3666 namespace.go:142] [default-tenant] updating namespace I0505 20:01:04.426949 3666 namespace.go:142] [image-controller] updating namespace I0505 20:01:04.465375 3666 namespace.go:142] [image-controller] updating namespace I0505 20:01:04.543429 3666 namespace.go:142] [build-service] updating namespace I0505 20:01:04.567031 3666 namespace.go:142] [integration-service] updating namespace I0505 20:01:04.590952 3666 namespace.go:98] [release-service] adding namespace I0505 20:01:04.591804 3666 namespace.go:102] [release-service] adding namespace took 835.445µs I0505 20:01:04.690250 3666 namespace.go:142] [konflux-ui] updating namespace I0505 20:01:04.746016 3666 namespace.go:142] [release-service] updating namespace I0505 20:01:04.746066 3666 namespace.go:98] [konflux-info] adding namespace I0505 20:01:04.746828 3666 namespace.go:102] [konflux-info] adding namespace took 749.606µs I0505 20:01:04.788227 3666 namespace.go:98] [namespace-lister] adding namespace I0505 20:01:04.788995 3666 namespace.go:102] [namespace-lister] adding namespace took 750.029µs I0505 20:01:04.828686 3666 namespace.go:142] [image-controller] updating namespace I0505 20:01:04.871786 3666 namespace.go:142] [release-service] updating namespace I0505 20:01:04.871825 3666 namespace.go:98] [enterprise-contract-service] adding namespace I0505 20:01:04.872602 3666 namespace.go:102] [enterprise-contract-service] adding namespace took 769.344µs I0505 20:01:04.965652 3666 namespace.go:142] [default-tenant] updating namespace I0505 20:01:05.128509 3666 namespace.go:142] [build-service] updating namespace I0505 20:01:05.128726 3666 namespace.go:98] [segment-bridge] adding namespace I0505 20:01:05.128763 3666 namespace.go:98] [konflux-cli] adding namespace I0505 20:01:05.129567 3666 namespace.go:102] [segment-bridge] adding namespace took 827.437µs I0505 20:01:05.130200 3666 namespace.go:102] [konflux-cli] adding namespace took 1.422021ms I0505 20:01:05.204552 3666 namespace.go:142] [konflux-info] updating namespace I0505 20:01:05.328087 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=63 I0505 20:01:05.400345 3666 namespace.go:142] [konflux-info] updating namespace I0505 20:01:05.457274 3666 namespace.go:142] [namespace-lister] updating namespace I0505 20:01:05.628435 3666 namespace.go:142] [namespace-lister] updating namespace I0505 20:01:05.743542 3666 namespace.go:142] [enterprise-contract-service] updating namespace I0505 20:01:05.816470 3666 namespace.go:142] [default-tenant] updating namespace I0505 20:01:05.934419 3666 namespace.go:142] [enterprise-contract-service] updating namespace I0505 20:01:05.993775 3666 namespace.go:142] [segment-bridge] updating namespace I0505 20:01:06.070248 3666 namespace.go:142] [segment-bridge] updating namespace I0505 20:01:06.119450 3666 namespace.go:142] [konflux-cli] updating namespace I0505 20:01:06.188282 3666 namespace.go:142] [konflux-cli] updating namespace I0505 20:01:06.708380 3666 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0505 20:01:06.709001 3666 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0505 20:01:06.709020 3666 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0505 20:01:06.711480 3666 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0505 20:01:06.749643 3666 namespace.go:142] [image-controller] updating namespace I0505 20:01:06.841060 3666 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0505 20:01:06.841088 3666 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0505 20:01:06.841878 3666 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0505 20:01:07.904130 3666 namespace.go:142] [integration-service] updating namespace I0505 20:01:08.105474 3666 namespace.go:142] [build-service] updating namespace I0505 20:01:08.153995 3666 namespace.go:142] [default-tenant] updating namespace I0505 20:01:08.180327 3666 namespace.go:142] [image-controller] updating namespace I0505 20:01:08.230308 3666 namespace.go:142] [integration-service] updating namespace I0505 20:01:08.347422 3666 namespace.go:142] [build-service] updating namespace I0505 20:01:08.444033 3666 namespace.go:142] [default-tenant] updating namespace I0505 20:01:08.527197 3666 namespace.go:142] [image-controller] updating namespace I0505 20:01:08.614135 3666 namespace.go:142] [integration-service] updating namespace I0505 20:01:10.360619 3666 namespace.go:142] [enterprise-contract-service] updating namespace W0505 20:01:10.632550 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0505 20:01:10.638923 3666 namespace.go:142] [konflux-cli] updating namespace I0505 20:01:10.828617 3666 namespace.go:142] [konflux-info] updating namespace I0505 20:01:11.028852 3666 namespace.go:142] [namespace-lister] updating namespace I0505 20:01:11.224559 3666 namespace.go:142] [release-service] updating namespace I0505 20:01:11.436056 3666 namespace.go:142] [segment-bridge] updating namespace I0505 20:01:11.561246 3666 namespace.go:142] [enterprise-contract-service] updating namespace I0505 20:01:11.581400 3666 namespace.go:142] [konflux-cli] updating namespace I0505 20:01:11.607490 3666 namespace.go:142] [konflux-info] updating namespace I0505 20:01:11.637901 3666 namespace.go:142] [namespace-lister] updating namespace I0505 20:01:11.659746 3666 namespace.go:142] [release-service] updating namespace I0505 20:01:11.676722 3666 namespace.go:142] [segment-bridge] updating namespace I0505 20:01:11.750840 3666 namespace.go:142] [enterprise-contract-service] updating namespace I0505 20:01:11.823404 3666 namespace.go:142] [konflux-cli] updating namespace I0505 20:01:11.900966 3666 namespace.go:142] [konflux-info] updating namespace I0505 20:01:11.977716 3666 namespace.go:142] [namespace-lister] updating namespace I0505 20:01:12.045258 3666 namespace.go:142] [release-service] updating namespace I0505 20:01:12.123976 3666 namespace.go:142] [segment-bridge] updating namespace W0505 20:01:13.633631 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0505 20:01:13.660374 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0505 20:01:15.649626 3666 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0505 20:01:17.672323 3666 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0505 20:01:20.932598 3666 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0505 20:01:20.932614 3666 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0505 20:01:21.709282 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0505 20:01:21.709373 3666 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0505 20:01:21.741324 3666 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0505 20:01:21.741341 3666 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0505 20:01:21.774475 3666 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0505 20:01:21.775383 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0505 20:01:21.829882 3666 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0505 20:01:21.830944 3666 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0505 20:01:23.327194 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=85 I0505 20:01:25.433983 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0505 20:01:33.086065 3666 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rdr5l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:01:33.086710 3666 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rdr5l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:01:33.800223 3666 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rdr5l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:01:33.800243 3666 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-rdr5l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:01:33.809362 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-rdr5l I0505 20:01:33.809388 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-rdr5l I0505 20:01:37.412567 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0505 20:01:58.327050 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=14 I0505 20:02:11.334359 3666 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=650 I0505 20:02:29.475430 3666 obj_retry.go:498] Detected object konflux-ui/proxy-5f69b8694b-llc59 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:02:29.476059 3666 obj_retry.go:498] Detected object konflux-ui/proxy-5f69b8694b-llc59 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:02:30.136898 3666 obj_retry.go:492] Detected object konflux-ui/proxy-5f69b8694b-llc59 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:02:30.136922 3666 obj_retry.go:492] Detected object konflux-ui/proxy-5f69b8694b-llc59 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:02:30.156668 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-5f69b8694b-llc59 I0505 20:02:30.156695 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-5f69b8694b-llc59 I0505 20:02:35.418146 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0505 20:02:37.486564 3666 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0505 20:02:37.487162 3666 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0505 20:02:37.507710 3666 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:02:37.507732 3666 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0505 20:02:37.515511 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0505 20:02:37.515551 3666 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0505 20:03:00.408030 3666 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0505 20:03:03.561455 3666 namespace.go:98] [managed-tenant-qbo52] adding namespace I0505 20:03:03.562358 3666 namespace.go:102] [managed-tenant-qbo52] adding namespace took 881.571µs I0505 20:03:03.614177 3666 namespace.go:142] [managed-tenant-qbo52] updating namespace I0505 20:03:03.633970 3666 namespace.go:142] [managed-tenant-qbo52] updating namespace I0505 20:03:04.633156 3666 namespace.go:142] [managed-tenant-qbo52] updating namespace I0505 20:03:04.758199 3666 namespace.go:142] [managed-tenant-qbo52] updating namespace I0505 20:03:04.858634 3666 namespace.go:142] [managed-tenant-qbo52] updating namespace I0505 20:03:32.450251 3666 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5