+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-152.ec2.internal ++ [[ -n ip-10-0-2-152.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-152.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0514 23:14:28.164819708 - disable conntrack on geneve port' I0514 23:14:28.164819708 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0514 23:14:28.183333653 - starting ovnkube-node' I0514 23:14:28.183333653 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-152.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-152.ec2.internal --init-node ip-10-0-2-152.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-152.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0514 23:14:28.225928 3796 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0514 23:14:28.225994 3796 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-bdaf609cfa.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0514 23:14:28.227379 3796 kube.go:431] Waiting for certificate I0514 23:14:28.227427 3796 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0514 23:14:28.227464 3796 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0514 23:14:28.228276 3796 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0514 23:14:28.254529 3796 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:28.254547 3796 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:28.257245 3796 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:28.259818 3796 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-9qn49" I0514 23:14:28.266946 3796 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-9qn49" I0514 23:14:28.267013 3796 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0514 23:14:29.227804 3796 kube.go:438] Certificate found I0514 23:14:29.228410 3796 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0514 23:14:29.229371 3796 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0514 23:14:29.273469 3796 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-05-15 23:09:28 +0000 UTC" deadline="2026-05-15 16:27:00.629482897 +0000 UTC" I0514 23:14:29.273523 3796 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="17h12m31.35597867s" I0514 23:14:29.305369 3796 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0514 23:14:29.316073 3796 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0514 23:14:29.316166 3796 server.go:101] MetricServer registers OVS metrics I0514 23:14:29.316186 3796 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0514 23:14:29.316246 3796 factory.go:561] Starting watch factory I0514 23:14:29.316328 3796 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.316344 3796 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.316397 3796 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.316461 3796 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.316494 3796 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.316526 3796 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.316573 3796 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.316633 3796 server.go:105] MetricServer registers OVN DB metrics I0514 23:14:29.316684 3796 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.316910 3796 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.316928 3796 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.317407 3796 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.317428 3796 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.321333 3796 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0514 23:14:29.321364 3796 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0514 23:14:29.322013 3796 controller_manager.go:370] Starting the ovnkube controller I0514 23:14:29.322023 3796 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-152.ec2.internal I0514 23:14:29.322064 3796 controller_manager.go:395] NBDB zone sync took: 34.258µs I0514 23:14:29.322072 3796 factory.go:561] Starting watch factory I0514 23:14:29.335200 3796 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0514 23:14:29.335219 3796 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0514 23:14:29.338478 3796 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.338726 3796 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.339110 3796 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.340258 3796 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.341399 3796 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.342714 3796 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0514 23:14:29.342795 3796 server.go:109] MetricServer registers OVN Controller metrics I0514 23:14:29.348122 3796 server.go:113] MetricServer registers OVN Northd metrics I0514 23:14:29.352803 3796 metrics.go:519] OVN Metrics Server starts to run ... I0514 23:14:29.353984 3796 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0514 23:14:29.421626 3796 factory.go:1977] *v1.Service informer cache synced successfully I0514 23:14:29.421657 3796 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0514 23:14:29.421661 3796 factory.go:1977] *v1.Pod informer cache synced successfully I0514 23:14:29.421664 3796 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0514 23:14:29.421667 3796 factory.go:1977] *v1.Namespace informer cache synced successfully I0514 23:14:29.421670 3796 factory.go:1977] *v1.Node informer cache synced successfully I0514 23:14:29.421754 3796 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:29.421761 3796 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:29.421781 3796 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:29.421770 3796 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:29.425062 3796 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:29.425062 3796 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0514 23:14:29.427071 3796 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0514 23:14:29.427082 3796 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0514 23:14:29.427134 3796 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.427146 3796 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.427762 3796 factory.go:1977] *v1.Node informer cache synced successfully I0514 23:14:29.427773 3796 factory.go:1977] *v1.Service informer cache synced successfully I0514 23:14:29.427777 3796 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0514 23:14:29.427779 3796 factory.go:1977] *v1.Pod informer cache synced successfully I0514 23:14:29.427782 3796 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0514 23:14:29.427785 3796 factory.go:1977] *v1.Namespace informer cache synced successfully I0514 23:14:29.429850 3796 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.432191 3796 factory.go:1977] *v1.EgressIP informer cache synced successfully I0514 23:14:29.432240 3796 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.432251 3796 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.432802 3796 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0514 23:14:29.432816 3796 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0514 23:14:29.434977 3796 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.438249 3796 factory.go:1977] *v1.EgressIP informer cache synced successfully I0514 23:14:29.438319 3796 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0514 23:14:29.438370 3796 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.438398 3796 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.440984 3796 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.444265 3796 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0514 23:14:29.444339 3796 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0514 23:14:29.444394 3796 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.444405 3796 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.447725 3796 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.449702 3796 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0514 23:14:29.449717 3796 factory.go:1977] *v1.EgressService informer cache synced successfully I0514 23:14:29.449774 3796 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.449790 3796 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.452742 3796 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.455242 3796 factory.go:1977] *v1.EgressService informer cache synced successfully I0514 23:14:29.455258 3796 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0514 23:14:29.455440 3796 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0514 23:14:29.455460 3796 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0514 23:14:29.458200 3796 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0514 23:14:29.460516 3796 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0514 23:14:29.460533 3796 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0514 23:14:29.460631 3796 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.460698 3796 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.460658 3796 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.460809 3796 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.463823 3796 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.463868 3796 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0514 23:14:29.465799 3796 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0514 23:14:29.465880 3796 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0514 23:14:29.465892 3796 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0514 23:14:29.465902 3796 factory.go:680] Watch Factory start up complete, took: 149.65819ms I0514 23:14:29.465979 3796 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0514 23:14:29.470879 3796 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0514 23:14:29.470891 3796 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0514 23:14:29.470897 3796 factory.go:680] Watch Factory start up complete, took: 148.823984ms I0514 23:14:29.470904 3796 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-152.ec2.internal" zone I0514 23:14:29.503378 3796 default_node_network_controller.go:710] Initializing the default node network controller I0514 23:14:29.565702 3796 udn_isolation.go:101] Starting UDN host isolation manager I0514 23:14:29.571415 3796 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0514 23:14:29.624850 3796 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0514 23:14:29.624928 3796 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0514 23:14:29.624948 3796 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0514 23:14:29.658430 3796 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0514 23:14:29.658491 3796 default_node_network_controller.go:809] Node ip-10-0-2-152.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0514 23:14:29.663967 3796 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.152"] k8s.ovn.org/zone-name:ip-10-0-2-152.ec2.internal] on node ip-10-0-2-152.ec2.internal I0514 23:14:29.686667 3796 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0514 23:14:29.692703 3796 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0514 23:14:29.692837 3796 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0514 23:14:29.692972 3796 gateway_init.go:263] Preparing Gateway I0514 23:14:29.692984 3796 gateway_shared_intf.go:1687] Creating new gateway I0514 23:14:29.721436 3796 controller_manager.go:429] Waiting for node in zone sync took: 250.519804ms I0514 23:14:29.727628 3796 controller_manager.go:611] Switching to transit router for layer2 networks I0514 23:14:29.727642 3796 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-152.ec2.internal I0514 23:14:29.744489 3796 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0514 23:14:29.750953 3796 controller_manager.go:290] SCTP support detected in OVN I0514 23:14:29.751449 3796 services_controller.go:69] Creating services controller for network=default I0514 23:14:29.751562 3796 udn_enabled_svc.go:80] Waiting for service informer to sync I0514 23:14:29.751628 3796 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0514 23:14:29.751663 3796 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0514 23:14:29.751677 3796 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0514 23:14:29.751685 3796 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0514 23:14:29.751880 3796 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0514 23:14:29.751932 3796 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0514 23:14:29.751967 3796 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 53.213µs I0514 23:14:29.751987 3796 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 44.071µs I0514 23:14:29.751991 3796 udn_enabled_svc.go:95] Performing full resync I0514 23:14:29.751997 3796 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 63.912µs I0514 23:14:29.752002 3796 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0514 23:14:29.752008 3796 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0514 23:14:29.752017 3796 default_network_controller.go:325] Starting the default network controller I0514 23:14:29.752136 3796 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 13.972µs I0514 23:14:29.752243 3796 udn_enabled_svc.go:99] Waiting for handler to sync I0514 23:14:29.752257 3796 udn_enabled_svc.go:104] Starting worker I0514 23:14:29.760945 3796 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0514 23:14:29.761001 3796 repair.go:33] Syncing exgw routes took 43.379µs I0514 23:14:29.761024 3796 default_network_controller.go:414] Starting all the Watchers... I0514 23:14:29.761153 3796 namespace.go:98] [openshift-authentication-operator] adding namespace I0514 23:14:29.761162 3796 namespace.go:98] [openshift-backplane-cee] adding namespace I0514 23:14:29.761166 3796 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0514 23:14:29.761177 3796 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0514 23:14:29.761178 3796 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0514 23:14:29.761186 3796 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0514 23:14:29.761188 3796 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0514 23:14:29.761190 3796 namespace.go:98] [kube-public] adding namespace I0514 23:14:29.761202 3796 namespace.go:98] [openshift-operators] adding namespace I0514 23:14:29.761205 3796 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0514 23:14:29.761205 3796 namespace.go:98] [openshift-must-gather-operator] adding namespace I0514 23:14:29.761204 3796 namespace.go:98] [openshift-backplane-tam] adding namespace I0514 23:14:29.761211 3796 namespace.go:98] [openshift-ingress-canary] adding namespace I0514 23:14:29.761215 3796 namespace.go:98] [openshift] adding namespace I0514 23:14:29.761220 3796 namespace.go:98] [openshift-config-operator] adding namespace I0514 23:14:29.761727 3796 namespace.go:102] [openshift-authentication-operator] adding namespace took 563.031µs I0514 23:14:29.761742 3796 namespace.go:98] [openshift-console-operator] adding namespace I0514 23:14:29.762418 3796 namespace.go:102] [openshift-console-operator] adding namespace took 666.463µs I0514 23:14:29.762437 3796 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0514 23:14:29.763385 3796 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 939.842µs I0514 23:14:29.763403 3796 namespace.go:98] [openshift-ingress-operator] adding namespace I0514 23:14:29.763973 3796 namespace.go:102] [openshift-backplane-cee] adding namespace took 2.798477ms I0514 23:14:29.764001 3796 namespace.go:98] [openshift-network-node-identity] adding namespace I0514 23:14:29.764445 3796 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 3.267463ms I0514 23:14:29.764460 3796 namespace.go:98] [openshift-dns-operator] adding namespace I0514 23:14:29.764954 3796 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 3.767742ms I0514 23:14:29.764972 3796 namespace.go:98] [openshift-operators-redhat] adding namespace I0514 23:14:29.765552 3796 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 4.360082ms I0514 23:14:29.765561 3796 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0514 23:14:29.766155 3796 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 4.964143ms I0514 23:14:29.766175 3796 namespace.go:98] [openshift-node] adding namespace I0514 23:14:29.766816 3796 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.615679ms I0514 23:14:29.766831 3796 namespace.go:98] [openshift-backplane] adding namespace I0514 23:14:29.767475 3796 namespace.go:102] [kube-public] adding namespace took 6.272119ms I0514 23:14:29.767491 3796 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0514 23:14:29.768116 3796 namespace.go:102] [openshift-operators] adding namespace took 6.902726ms I0514 23:14:29.768135 3796 namespace.go:98] [kube-node-lease] adding namespace I0514 23:14:29.768656 3796 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.444461ms I0514 23:14:29.768674 3796 namespace.go:98] [openshift-apiserver-operator] adding namespace I0514 23:14:29.769149 3796 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.935211ms I0514 23:14:29.769171 3796 namespace.go:98] [openshift-network-diagnostics] adding namespace I0514 23:14:29.769722 3796 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.50491ms I0514 23:14:29.769743 3796 namespace.go:98] [openshift-console-user-settings] adding namespace I0514 23:14:29.770244 3796 namespace.go:102] [openshift-ingress-canary] adding namespace took 9.023654ms I0514 23:14:29.770267 3796 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0514 23:14:29.770751 3796 namespace.go:102] [openshift] adding namespace took 9.526269ms I0514 23:14:29.770767 3796 namespace.go:98] [openshift-package-operator] adding namespace I0514 23:14:29.771232 3796 namespace.go:102] [openshift-config-operator] adding namespace took 10.00259ms I0514 23:14:29.771252 3796 namespace.go:98] [openshift-machine-config-operator] adding namespace I0514 23:14:29.771740 3796 namespace.go:102] [openshift-ingress-operator] adding namespace took 8.33006ms I0514 23:14:29.771756 3796 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0514 23:14:29.772129 3796 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.121259ms I0514 23:14:29.772146 3796 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0514 23:14:29.772550 3796 namespace.go:102] [openshift-dns-operator] adding namespace took 8.082974ms I0514 23:14:29.772566 3796 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0514 23:14:29.773019 3796 namespace.go:102] [openshift-operators-redhat] adding namespace took 8.040726ms I0514 23:14:29.773038 3796 namespace.go:98] [openshift-backplane-csm] adding namespace I0514 23:14:29.773407 3796 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.839493ms I0514 23:14:29.773425 3796 namespace.go:98] [openshift-authentication] adding namespace I0514 23:14:29.774183 3796 namespace.go:102] [openshift-node] adding namespace took 8.001493ms I0514 23:14:29.774199 3796 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0514 23:14:29.774621 3796 namespace.go:102] [openshift-backplane] adding namespace took 7.766068ms I0514 23:14:29.774639 3796 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0514 23:14:29.775074 3796 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.57806ms I0514 23:14:29.775087 3796 namespace.go:98] [dedicated-admin] adding namespace I0514 23:14:29.775472 3796 namespace.go:102] [kube-node-lease] adding namespace took 7.330401ms I0514 23:14:29.775490 3796 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace W0514 23:14:29.775876 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-kjlnq: pod openshift-network-diagnostics/network-check-target-kjlnq: no pod IPs found W0514 23:14:29.775897 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-snbfj: pod openshift-network-diagnostics/network-check-target-snbfj: no pod IPs found I0514 23:14:29.775905 3796 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.222844ms I0514 23:14:29.775921 3796 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0514 23:14:29.776416 3796 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.236206ms I0514 23:14:29.776430 3796 namespace.go:98] [openshift-ingress] adding namespace I0514 23:14:29.776933 3796 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.180276ms I0514 23:14:29.776950 3796 namespace.go:98] [openshift-kube-apiserver] adding namespace I0514 23:14:29.777419 3796 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.141644ms I0514 23:14:29.777431 3796 namespace.go:98] [openshift-multus] adding namespace I0514 23:14:29.777776 3796 namespace.go:102] [openshift-package-operator] adding namespace took 7.002263ms I0514 23:14:29.777787 3796 namespace.go:98] [openshift-config] adding namespace I0514 23:14:29.778258 3796 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.000281ms I0514 23:14:29.778271 3796 namespace.go:98] [openshift-backplane-ai-agent] adding namespace I0514 23:14:29.778684 3796 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.921977ms I0514 23:14:29.779121 3796 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.968579ms I0514 23:14:29.779134 3796 namespace.go:98] [openshift-insights] adding namespace I0514 23:14:29.779529 3796 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.955297ms I0514 23:14:29.779546 3796 namespace.go:98] [openshift-image-registry] adding namespace I0514 23:14:29.779989 3796 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.9439ms I0514 23:14:29.780007 3796 namespace.go:98] [openshift-backplane-srep] adding namespace I0514 23:14:29.780345 3796 namespace.go:102] [openshift-authentication] adding namespace took 6.913582ms I0514 23:14:29.780361 3796 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0514 23:14:29.780761 3796 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.556251ms I0514 23:14:29.780779 3796 namespace.go:98] [openshift-dns] adding namespace I0514 23:14:29.781292 3796 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.647136ms I0514 23:14:29.781307 3796 namespace.go:98] [openshift-monitoring] adding namespace I0514 23:14:29.781734 3796 namespace.go:102] [dedicated-admin] adding namespace took 6.639444ms I0514 23:14:29.781752 3796 namespace.go:98] [openshift-etcd] adding namespace I0514 23:14:29.782091 3796 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.591028ms I0514 23:14:29.782106 3796 namespace.go:98] [openshift-network-operator] adding namespace I0514 23:14:29.782582 3796 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.652998ms I0514 23:14:29.782596 3796 namespace.go:98] [openshift-customer-monitoring] adding namespace I0514 23:14:29.783192 3796 namespace.go:102] [openshift-ingress] adding namespace took 6.756818ms I0514 23:14:29.783207 3796 namespace.go:98] [openshift-infra] adding namespace I0514 23:14:29.783628 3796 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.674787ms W0514 23:14:29.783631 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-5t9hc: pod openshift-multus/network-metrics-daemon-5t9hc: no pod IPs found I0514 23:14:29.783639 3796 namespace.go:98] [openshift-ovn-kubernetes] adding namespace W0514 23:14:29.783645 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-j7h6h: pod openshift-multus/network-metrics-daemon-j7h6h: no pod IPs found I0514 23:14:29.784092 3796 namespace.go:102] [openshift-multus] adding namespace took 6.655934ms I0514 23:14:29.784100 3796 namespace.go:98] [openshift-kube-scheduler] adding namespace I0514 23:14:29.784562 3796 namespace.go:102] [openshift-config] adding namespace took 6.770498ms I0514 23:14:29.784576 3796 namespace.go:98] [openshift-backplane-mobb] adding namespace I0514 23:14:29.785014 3796 namespace.go:102] [openshift-backplane-ai-agent] adding namespace took 6.739381ms I0514 23:14:29.785027 3796 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0514 23:14:29.785419 3796 namespace.go:102] [openshift-insights] adding namespace took 6.277883ms I0514 23:14:29.785433 3796 namespace.go:98] [openshift-network-console] adding namespace I0514 23:14:29.785929 3796 namespace.go:102] [openshift-image-registry] adding namespace took 6.377471ms I0514 23:14:29.785942 3796 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0514 23:14:29.786337 3796 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.325106ms I0514 23:14:29.786349 3796 namespace.go:98] [openshift-console] adding namespace I0514 23:14:29.786722 3796 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.356749ms I0514 23:14:29.786737 3796 namespace.go:98] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] adding namespace I0514 23:14:29.787236 3796 namespace.go:102] [openshift-dns] adding namespace took 6.451424ms I0514 23:14:29.787250 3796 namespace.go:98] [openshift-cluster-version] adding namespace I0514 23:14:29.787692 3796 namespace.go:102] [openshift-monitoring] adding namespace took 6.380199ms I0514 23:14:29.787708 3796 namespace.go:98] [kube-system] adding namespace I0514 23:14:29.788066 3796 namespace.go:102] [openshift-etcd] adding namespace took 6.309063ms I0514 23:14:29.788075 3796 namespace.go:98] [openshift-marketplace] adding namespace I0514 23:14:29.788584 3796 namespace.go:102] [openshift-network-operator] adding namespace took 6.469198ms I0514 23:14:29.788599 3796 namespace.go:98] [openshift-backplane-cse] adding namespace I0514 23:14:29.789000 3796 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.399665ms I0514 23:14:29.789012 3796 namespace.go:98] [openshift-service-ca-operator] adding namespace I0514 23:14:29.789464 3796 namespace.go:102] [openshift-infra] adding namespace took 6.250937ms I0514 23:14:29.789476 3796 namespace.go:98] [openshift-logging] adding namespace I0514 23:14:29.789842 3796 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.19728ms I0514 23:14:29.789856 3796 namespace.go:98] [openshift-controller-manager] adding namespace I0514 23:14:29.790334 3796 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.229442ms I0514 23:14:29.790347 3796 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0514 23:14:29.790686 3796 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.105069ms I0514 23:14:29.790701 3796 namespace.go:98] [openshift-machine-api] adding namespace I0514 23:14:29.791191 3796 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.158689ms I0514 23:14:29.791205 3796 namespace.go:98] [openshift-config-managed] adding namespace I0514 23:14:29.791675 3796 namespace.go:102] [openshift-network-console] adding namespace took 6.237678ms I0514 23:14:29.791686 3796 namespace.go:98] [openshift-apiserver] adding namespace I0514 23:14:29.792181 3796 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.234237ms I0514 23:14:29.792727 3796 namespace.go:102] [openshift-console] adding namespace took 6.372394ms I0514 23:14:29.793184 3796 namespace.go:102] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] adding namespace took 6.441423ms I0514 23:14:29.793535 3796 namespace.go:102] [openshift-cluster-version] adding namespace took 6.280625ms W0514 23:14:29.793566 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-p6k99: pod kube-system/global-pull-secret-syncer-p6k99: no pod IPs found W0514 23:14:29.793578 3796 base_network_controller_namespace.go:410] Failed to get IPs for pod kube-system/global-pull-secret-syncer-gjtjf: pod kube-system/global-pull-secret-syncer-gjtjf: no pod IPs found I0514 23:14:29.794076 3796 namespace.go:102] [kube-system] adding namespace took 6.360997ms I0514 23:14:29.794095 3796 namespace.go:98] [openshift-host-network] adding namespace I0514 23:14:29.794463 3796 namespace.go:102] [openshift-marketplace] adding namespace took 6.381617ms I0514 23:14:29.794897 3796 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.278391ms I0514 23:14:29.794916 3796 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0514 23:14:29.795352 3796 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.336013ms I0514 23:14:29.795717 3796 namespace.go:102] [openshift-logging] adding namespace took 6.236067ms I0514 23:14:29.796174 3796 namespace.go:102] [openshift-controller-manager] adding namespace took 6.312142ms I0514 23:14:29.796578 3796 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.224898ms I0514 23:14:29.796596 3796 namespace.go:98] [default] adding namespace I0514 23:14:29.797062 3796 namespace.go:102] [openshift-machine-api] adding namespace took 6.35532ms I0514 23:14:29.797471 3796 namespace.go:102] [openshift-config-managed] adding namespace took 6.259139ms I0514 23:14:29.797489 3796 namespace.go:98] [openshift-route-controller-manager] adding namespace I0514 23:14:29.797939 3796 namespace.go:102] [openshift-apiserver] adding namespace took 6.245428ms I0514 23:14:29.798450 3796 namespace.go:102] [openshift-host-network] adding namespace took 4.348759ms I0514 23:14:29.798906 3796 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 3.981753ms I0514 23:14:29.799439 3796 namespace.go:102] [default] adding namespace took 2.835877ms I0514 23:14:29.800289 3796 namespace.go:102] [openshift-route-controller-manager] adding namespace took 2.79414ms W0514 23:14:29.800368 3796 master.go:458] Unable to parse local node ip-10-0-2-152.ec2.internal chassis-id annotation. Chassis may be removed during sync W0514 23:14:29.800377 3796 master.go:471] Unable to parse remote node ip-10-0-0-78.ec2.internal chassis-id annotation. Chassis may be removed during sync I0514 23:14:29.800382 3796 master.go:481] Removing stale chassis with ID/Name: ef851572-e71e-4489-a413-9d3edf1ba10c, hostname: ip-10-0-2-152.ec2.internal I0514 23:14:29.800845 3796 master.go:548] Adding or Updating local node "ip-10-0-2-152.ec2.internal" for network "default" I0514 23:14:29.800915 3796 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-78.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-78.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-78.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-78.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-78.ec2.internal I0514 23:14:29.802141 3796 zone_ic_handler.go:195] Time taken to create transit switch: 567.244µs I0514 23:14:29.802162 3796 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-98.ec2.internal for the network default I0514 23:14:29.803257 3796 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-98.ec2.internal" on network "default" took: 1.689925ms I0514 23:14:29.804411 3796 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-152.ec2.internal for the network default I0514 23:14:29.804650 3796 zone_ic_handler.go:195] Time taken to create transit switch: 226.247µs I0514 23:14:29.805432 3796 default_network_controller.go:823] Node add failed for ip-10-0-2-152.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-152.ec2.internal, error getting gateway config for node ip-10-0-2-152.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-152.ec2.internal", failed to update chassis to local for local node ip-10-0-2-152.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-152.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-152.ec2.internal] E0514 23:14:29.805480 3796 obj_retry.go:575] Failed to create *v1.Node ip-10-0-2-152.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-152.ec2.internal, error getting gateway config for node ip-10-0-2-152.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-152.ec2.internal", failed to update chassis to local for local node ip-10-0-2-152.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-152.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-152.ec2.internal] I0514 23:14:29.805525 3796 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0514 23:14:29.805544 3796 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0514 23:14:29.805545 3796 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-2-152.ec2.internal", UID:"2f583bbd-05df-4db9-8f77-b4bb90e73ad6", APIVersion:"v1", ResourceVersion:"9170", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-152.ec2.internal, error getting gateway config for node ip-10-0-2-152.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-152.ec2.internal", failed to update chassis to local for local node ip-10-0-2-152.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-152.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-152.ec2.internal] I0514 23:14:29.805558 3796 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0514 23:14:29.805561 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:29.805590 3796 node_tracker.go:235] Node ip-10-0-0-78.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-78.ec2.internal" I0514 23:14:29.805624 3796 node_tracker.go:169] Node ip-10-0-0-78.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:29.805638 3796 services_controller.go:574] Full service sync requested for network=default I0514 23:14:29.805643 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-98.ec2.internal in network "default" I0514 23:14:29.805684 3796 node_tracker.go:169] Node ip-10-0-1-98.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:29.805692 3796 services_controller.go:574] Full service sync requested for network=default I0514 23:14:29.805700 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-152.ec2.internal in network "default" I0514 23:14:29.805707 3796 node_tracker.go:235] Node ip-10-0-2-152.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-152.ec2.internal" I0514 23:14:29.805713 3796 node_tracker.go:169] Node ip-10-0-2-152.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:29.805718 3796 services_controller.go:574] Full service sync requested for network=default I0514 23:14:29.818637 3796 iptables.go:108] Creating table: filter chain: FORWARD I0514 23:14:29.825425 3796 iptables.go:108] Creating table: filter chain: OUTPUT I0514 23:14:29.832252 3796 gateway_shared_intf.go:1816] Gateway Creation Complete I0514 23:14:29.832269 3796 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-152.ec2.internal","mac-address":"0e:50:d4:c8:7a:5b","ip-addresses":["10.0.2.152/24"],"ip-address":"10.0.2.152/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:ef851572-e71e-4489-a413-9d3edf1ba10c k8s.ovn.org/node-encap-ips:["10.0.2.152"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.152/24"} k8s.ovn.org/zone-name:ip-10-0-2-152.ec2.internal] on node ip-10-0-2-152.ec2.internal I0514 23:14:29.848564 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-152.ec2.internal in network "default" I0514 23:14:29.848572 3796 master.go:548] Adding or Updating local node "ip-10-0-2-152.ec2.internal" for network "default" I0514 23:14:29.848634 3796 node_tracker.go:169] Node ip-10-0-2-152.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:29.848648 3796 services_controller.go:574] Full service sync requested for network=default I0514 23:14:29.852714 3796 default_node_network_controller.go:519] Node connection status = connected W0514 23:14:29.853365 3796 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-j7h6h: pod openshift-multus/network-metrics-daemon-j7h6h: no pod IPs found W0514 23:14:29.853392 3796 gateway.go:207] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-p6k99: pod kube-system/global-pull-secret-syncer-p6k99: no pod IPs found W0514 23:14:29.853403 3796 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-snbfj: pod openshift-network-diagnostics/network-check-target-snbfj: no pod IPs found I0514 23:14:29.854434 3796 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-152.ec2.internal for the network default I0514 23:14:29.854637 3796 zone_ic_handler.go:195] Time taken to create transit switch: 190.546µs I0514 23:14:29.906153 3796 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0514 23:14:29.906175 3796 services_controller.go:208] Setting up event handlers for services for network=default I0514 23:14:29.906239 3796 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0514 23:14:29.906283 3796 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0514 23:14:29.906298 3796 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0514 23:14:30.006889 3796 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0514 23:14:30.006910 3796 repair.go:60] Starting repairing loop for services I0514 23:14:30.006993 3796 repair.go:131] Deleted 0 stale service LBs I0514 23:14:30.007015 3796 repair.go:137] Deleted 0 stale Chassis Template Vars I0514 23:14:30.007033 3796 repair.go:62] Finished repairing loop for services: 125.171µs I0514 23:14:30.007054 3796 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0514 23:14:30.007063 3796 services_controller.go:254] Starting workers for network=default I0514 23:14:30.007423 3796 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-p6k99] creating logical port kube-system_global-pull-secret-syncer-p6k99 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:30.007530 3796 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-snbfj] creating logical port openshift-network-diagnostics_network-check-target-snbfj for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:30.007554 3796 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-kjlnq, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-kjlnq: suppressed error logged: pod openshift-network-diagnostics/network-check-target-kjlnq: no pod IPs found I0514 23:14:30.007568 3796 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-p6k99 I0514 23:14:30.007582 3796 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-j7h6h] creating logical port openshift-multus_network-metrics-daemon-j7h6h for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:30.007649 3796 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-snbfj I0514 23:14:30.007664 3796 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-j7h6h I0514 23:14:30.007694 3796 obj_retry.go:578] Failed to create *v1.Pod kube-system/global-pull-secret-syncer-gjtjf, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-gjtjf: suppressed error logged: pod kube-system/global-pull-secret-syncer-gjtjf: no pod IPs found I0514 23:14:30.024556 3796 pod.go:62] [kube-system/global-pull-secret-syncer-p6k99] pod update took 17.045895ms I0514 23:14:30.024576 3796 base_network_controller_pods.go:951] [default/kube-system/global-pull-secret-syncer-p6k99] addLogicalPort annotation time took 17.06894ms I0514 23:14:30.025411 3796 pods.go:271] [kube-system/global-pull-secret-syncer-p6k99] addLogicalPort took 17.997762ms, libovsdb time 572.818µs I0514 23:14:30.025451 3796 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-5t9hc, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-5t9hc: suppressed error logged: pod openshift-multus/network-metrics-daemon-5t9hc: no pod IPs found I0514 23:14:30.026467 3796 pod.go:62] [openshift-multus/network-metrics-daemon-j7h6h] pod update took 18.82701ms I0514 23:14:30.026483 3796 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-j7h6h] addLogicalPort annotation time took 18.843568ms I0514 23:14:30.027371 3796 pods.go:271] [openshift-multus/network-metrics-daemon-j7h6h] addLogicalPort took 19.794617ms, libovsdb time 578.529µs I0514 23:14:30.027889 3796 pod.go:62] [openshift-network-diagnostics/network-check-target-snbfj] pod update took 20.288506ms I0514 23:14:30.027904 3796 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-snbfj] addLogicalPort annotation time took 20.304268ms I0514 23:14:30.028632 3796 pods.go:271] [openshift-network-diagnostics/network-check-target-snbfj] addLogicalPort took 21.111302ms, libovsdb time 512.605µs I0514 23:14:30.029182 3796 admin_network_policy_controller.go:227] Starting controller default-network-controller I0514 23:14:30.029222 3796 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0514 23:14:30.029236 3796 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0514 23:14:30.029240 3796 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0514 23:14:30.029413 3796 repair.go:29] Repairing admin network policies took 168.713µs I0514 23:14:30.029550 3796 repair.go:92] Repairing baseline admin network policies took 128.3µs I0514 23:14:30.030083 3796 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0514 23:14:30.030098 3796 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0514 23:14:30.031674 3796 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0514 23:14:30.031705 3796 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0514 23:14:30.031717 3796 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0514 23:14:30.032749 3796 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0514 23:14:30.032771 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0514 23:14:30.032780 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0514 23:14:30.034403 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0514 23:14:30.034423 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0514 23:14:30.034432 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0514 23:14:30.035085 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0514 23:14:30.035106 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0514 23:14:30.035114 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0514 23:14:30.036899 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0514 23:14:30.036920 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0514 23:14:30.036927 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0514 23:14:30.038397 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0514 23:14:30.038416 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0514 23:14:30.038427 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0514 23:14:30.039485 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0514 23:14:30.039504 3796 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0514 23:14:30.039519 3796 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0514 23:14:30.041096 3796 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0514 23:14:30.041114 3796 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0514 23:14:30.041124 3796 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0514 23:14:30.041705 3796 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0514 23:14:30.041726 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0514 23:14:30.041746 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0514 23:14:30.043080 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0514 23:14:30.043100 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0514 23:14:30.043109 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0514 23:14:30.043671 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0514 23:14:30.043689 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0514 23:14:30.043698 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0514 23:14:30.045343 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0514 23:14:30.045363 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0514 23:14:30.045371 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0514 23:14:30.046803 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0514 23:14:30.046821 3796 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0514 23:14:30.046830 3796 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0514 23:14:30.048002 3796 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0514 23:14:30.048021 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0514 23:14:30.048035 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0514 23:14:30.048633 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0514 23:14:30.048649 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0514 23:14:30.048657 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0514 23:14:30.049306 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0514 23:14:30.049326 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0514 23:14:30.049334 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0514 23:14:30.049888 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0514 23:14:30.049905 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0514 23:14:30.050721 3796 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0514 23:14:30.050742 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0514 23:14:30.051499 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0514 23:14:30.051520 3796 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0514 23:14:30.051528 3796 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0514 23:14:30.052746 3796 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0514 23:14:30.052762 3796 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0514 23:14:30.052771 3796 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0514 23:14:30.053990 3796 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0514 23:14:30.054008 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0514 23:14:30.054018 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0514 23:14:30.054969 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0514 23:14:30.054990 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0514 23:14:30.055000 3796 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0514 23:14:30.056126 3796 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0514 23:14:30.056145 3796 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0514 23:14:30.056153 3796 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0514 23:14:30.057778 3796 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0514 23:14:30.057794 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0514 23:14:30.057805 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0514 23:14:30.058276 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0514 23:14:30.058294 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0514 23:14:30.058303 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0514 23:14:30.058956 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0514 23:14:30.058971 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0514 23:14:30.058980 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0514 23:14:30.059403 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0514 23:14:30.059421 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0514 23:14:30.059427 3796 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0514 23:14:30.059664 3796 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0514 23:14:30.059677 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0514 23:14:30.059697 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0514 23:14:30.060368 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0514 23:14:30.060382 3796 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0514 23:14:30.060391 3796 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0514 23:14:30.060742 3796 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0514 23:14:30.066535 3796 egressfirewall.go:374] Starting EgressFirewall controller I0514 23:14:30.066571 3796 controller.go:133] Adding controller egress-firewall-controller event handlers I0514 23:14:30.066581 3796 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0514 23:14:30.066637 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0514 23:14:30.066653 3796 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0514 23:14:30.066640 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0514 23:14:30.066666 3796 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0514 23:14:30.066950 3796 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0514 23:14:30.066978 3796 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0514 23:14:30.066984 3796 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0514 23:14:30.066992 3796 egressqos.go:193] Setting up event handlers for EgressQoS I0514 23:14:30.067057 3796 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:30.067071 3796 egressfirewall.go:723] Syncing node "ip-10-0-1-98.ec2.internal" for egress firewall I0514 23:14:30.067078 3796 egressfirewall.go:723] Syncing node "ip-10-0-2-152.ec2.internal" for egress firewall I0514 23:14:30.067178 3796 egressqos.go:245] Starting EgressQoS Controller I0514 23:14:30.067189 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0514 23:14:30.067195 3796 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0514 23:14:30.067200 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0514 23:14:30.067205 3796 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0514 23:14:30.067210 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0514 23:14:30.067214 3796 shared_informer.go:356] "Caches are synced" controller="egressqos" I0514 23:14:30.067218 3796 egressqos.go:259] Repairing EgressQoSes I0514 23:14:30.067222 3796 egressqos.go:399] Starting repairing loop for egressqos I0514 23:14:30.067300 3796 egressqos.go:401] Finished repairing loop for egressqos: 77.974µs I0514 23:14:30.067317 3796 egressservice_zone.go:132] Setting up event handlers for Egress Services I0514 23:14:30.067343 3796 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-152.ec2.internal I0514 23:14:30.067379 3796 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-152.ec2.internal : 35.887µs I0514 23:14:30.067428 3796 egressservice_zone.go:210] Starting Egress Services Controller I0514 23:14:30.067457 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0514 23:14:30.067465 3796 shared_informer.go:356] "Caches are synced" controller="egressservices" I0514 23:14:30.067470 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0514 23:14:30.067475 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0514 23:14:30.067480 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0514 23:14:30.067484 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0514 23:14:30.067538 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0514 23:14:30.067545 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0514 23:14:30.067549 3796 egressservice_zone.go:228] Repairing Egress Services I0514 23:14:30.067842 3796 master_controller.go:88] Starting Admin Policy Based Route Controller I0514 23:14:30.067854 3796 external_controller.go:278] Starting Admin Policy Based Route Controller I0514 23:14:30.067866 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:30.067879 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 15.172µs I0514 23:14:30.067886 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-98.ec2.internal I0514 23:14:30.067894 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-98.ec2.internal: 8.632µs I0514 23:14:30.067899 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-152.ec2.internal I0514 23:14:30.067902 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-152.ec2.internal: 3.461µs I0514 23:14:30.068066 3796 default_network_controller.go:585] Completing all the Watchers took 307.031724ms I0514 23:14:30.068082 3796 default_network_controller.go:589] Starting unidling controllers I0514 23:14:30.068093 3796 unidle.go:46] Registering OVN SB ControllerEvent handler I0514 23:14:30.068099 3796 unidle.go:63] Populating Initial ContollerEvent events I0514 23:14:30.068111 3796 unidle.go:79] Setting up event handlers for services I0514 23:14:30.068177 3796 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0514 23:14:30.353088 3796 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:30.853983 3796 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:31.354090 3796 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:31.813876 3796 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0514 23:14:31.853931 3796 default_node_network_controller.go:519] Node connection status = connected I0514 23:14:31.872300 3796 bridgeconfig.go:537] Gateway is ready I0514 23:14:31.872318 3796 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0514 23:14:31.884088 3796 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.152. Updating OVN encap IP. I0514 23:14:31.889508 3796 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.152 - it is already configured I0514 23:14:31.889524 3796 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.152/24:{} 172.20.0.1/32:{}]. Updating annotations. I0514 23:14:31.889795 3796 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.152/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-152.ec2.internal","mac-address":"0e:50:d4:c8:7a:5b","ip-addresses":["10.0.2.152/24"],"ip-address":"10.0.2.152/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:ef851572-e71e-4489-a413-9d3edf1ba10c k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.152/24"}] on node ip-10-0-2-152.ec2.internal I0514 23:14:31.906986 3796 master.go:548] Adding or Updating local node "ip-10-0-2-152.ec2.internal" for network "default" I0514 23:14:31.907021 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-152.ec2.internal in network "default" I0514 23:14:31.907024 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-152.ec2.internal I0514 23:14:31.907041 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-152.ec2.internal: 24.63µs I0514 23:14:31.907054 3796 egressfirewall.go:723] Syncing node "ip-10-0-2-152.ec2.internal" for egress firewall I0514 23:14:31.907078 3796 node_tracker.go:169] Node ip-10-0-2-152.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:31.907096 3796 services_controller.go:574] Full service sync requested for network=default I0514 23:14:31.907451 3796 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.152 I0514 23:14:31.907569 3796 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-152.ec2.internal I0514 23:14:31.911890 3796 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-152.ec2.internal. Updating no re-route policies I0514 23:14:31.911969 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ns6ln I0514 23:14:31.911982 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-c8f4m I0514 23:14:31.911989 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ns6ln I0514 23:14:31.911989 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-wg2lj I0514 23:14:31.911998 3796 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-gjtjf I0514 23:14:31.912004 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-wg2lj I0514 23:14:31.912007 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-kjlnq I0514 23:14:31.912013 3796 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-gjtjf I0514 23:14:31.912016 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-kjlnq I0514 23:14:31.912019 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ns6ln after 0 failed attempt(s) I0514 23:14:31.912020 3796 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-vlmdq I0514 23:14:31.911969 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-snbfj I0514 23:14:31.912034 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-wg2lj after 0 failed attempt(s) I0514 23:14:31.912037 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-snbfj I0514 23:14:31.912035 3796 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-vlmdq I0514 23:14:31.912040 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-5kkk9 I0514 23:14:31.912043 3796 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-p6k99 I0514 23:14:31.912051 3796 obj_retry.go:409] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-gjtjf, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-gjtjf: suppressed error logged: pod kube-system/global-pull-secret-syncer-gjtjf: no pod IPs found I0514 23:14:31.912053 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-9rl7j I0514 23:14:31.912058 3796 obj_retry.go:398] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-p6k99 I0514 23:14:31.912066 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-9rl7j I0514 23:14:31.912066 3796 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-snbfj] creating logical port openshift-network-diagnostics_network-check-target-snbfj for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:31.912066 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-152.ec2.internal I0514 23:14:31.912070 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-w66b9 I0514 23:14:31.912069 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-694kt I0514 23:14:31.912080 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-152.ec2.internal I0514 23:14:31.912076 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-5t9hc I0514 23:14:31.912090 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-9rl7j after 0 failed attempt(s) I0514 23:14:31.912088 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-694kt I0514 23:14:31.912092 3796 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-152.ec2.internal I0514 23:14:31.912103 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-152.ec2.internal after 0 failed attempt(s) I0514 23:14:31.912111 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-j7h6h I0514 23:14:31.912113 3796 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-152.ec2.internal I0514 23:14:31.912131 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-j7h6h I0514 23:14:31.912135 3796 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-9q6jf I0514 23:14:31.912151 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-9q6jf I0514 23:14:31.912152 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-w66b9 I0514 23:14:31.912173 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-9q6jf after 0 failed attempt(s) I0514 23:14:31.912180 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-w66b9 after 0 failed attempt(s) I0514 23:14:31.912183 3796 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-j7h6h] creating logical port openshift-multus_network-metrics-daemon-j7h6h for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:31.912055 3796 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-kjlnq, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-kjlnq: suppressed error logged: pod openshift-network-diagnostics/network-check-target-kjlnq: no pod IPs found I0514 23:14:31.912151 3796 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-152.ec2.internal after 0 failed attempt(s) I0514 23:14:31.912117 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-694kt after 0 failed attempt(s) I0514 23:14:31.912099 3796 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-p6k99] creating logical port kube-system_global-pull-secret-syncer-p6k99 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:31.911994 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-c8f4m I0514 23:14:31.912236 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-c8f4m after 0 failed attempt(s) I0514 23:14:31.912059 3796 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-vlmdq after 0 failed attempt(s) I0514 23:14:31.912151 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-5t9hc I0514 23:14:31.912267 3796 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-5t9hc, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-5t9hc: suppressed error logged: pod openshift-multus/network-metrics-daemon-5t9hc: no pod IPs found I0514 23:14:31.912057 3796 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-5kkk9 I0514 23:14:31.912317 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-5kkk9 after 0 failed attempt(s) I0514 23:14:31.912773 3796 pods.go:271] [openshift-network-diagnostics/network-check-target-snbfj] addLogicalPort took 714.801µs, libovsdb time 330.134µs I0514 23:14:31.912785 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-snbfj after 0 failed attempt(s) I0514 23:14:31.913026 3796 pods.go:271] [kube-system/global-pull-secret-syncer-p6k99] addLogicalPort took 940.153µs, libovsdb time 302.957µs I0514 23:14:31.913035 3796 obj_retry.go:418] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-p6k99 after 0 failed attempt(s) I0514 23:14:31.913381 3796 pods.go:271] [openshift-multus/network-metrics-daemon-j7h6h] addLogicalPort took 1.230929ms, libovsdb time 220.23µs I0514 23:14:31.913387 3796 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-j7h6h after 0 failed attempt(s) I0514 23:14:31.923865 3796 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0514 23:14:31.936725 3796 iptables.go:108] Creating table: mangle chain: OUTPUT I0514 23:14:31.940382 3796 iptables.go:108] Creating table: nat chain: OUTPUT I0514 23:14:31.982419 3796 iptables.go:108] Creating table: nat chain: PREROUTING I0514 23:14:32.043199 3796 iptables.go:108] Creating table: filter chain: FORWARD I0514 23:14:32.061971 3796 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0514 23:14:32.062085 3796 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0514 23:14:32.062136 3796 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0514 23:14:32.062281 3796 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0514 23:14:32.062394 3796 gateway_init.go:325] Gateway and management port readiness took 2.213940603s I0514 23:14:32.062429 3796 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0514 23:14:32.062471 3796 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0514 23:14:32.062483 3796 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0514 23:14:32.062491 3796 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0514 23:14:32.062566 3796 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0514 23:14:32.062669 3796 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 71.972µs I0514 23:14:32.062941 3796 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0514 23:14:32.062966 3796 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.731µs I0514 23:14:32.062974 3796 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 34.052µs I0514 23:14:32.062980 3796 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0514 23:14:32.062988 3796 nad_controller.go:379] [node-nad-controller NAD controller]: started I0514 23:14:32.062993 3796 default_node_network_controller.go:899] Starting the default node network controller I0514 23:14:32.062997 3796 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0514 23:14:32.063020 3796 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.285µs I0514 23:14:32.063166 3796 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0514 23:14:32.063174 3796 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0514 23:14:32.063198 3796 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000d1c150 0xc0002b7d50 0xc0002b7dc0 0xc0002b7e30 0xc0002b7ea0 0xc0002b7f10 0xc0002ca070 0xc000471d60 0xc0002ca0e0 0xc0002ca150 0xc000b822d0 0xc0061e7500 0xc0002ca1c0 map[0x2ff9080:0xc00249e120 0x2ff95c0:0xc00249e420 0x2ffa2e0:0xc00249e060 0x2ffa580:0xc00249e0c0 0x2ffad60:0xc00249e180 0x2ffb000:0xc00249e1e0 0x2ffb2a0:0xc00249e3c0 0x2ffba80:0xc00249e480 0x2ffbd20:0xc0068a4000 0x3026980:0xc001097200 0x3026d20:0xc001097260 0x3027800:0xc000d42660 0x3029160:0xc001bca000 0x30298a0:0xc00249e000 0x302a720:0xc001096de0] 0xc00047e540 0} I0514 23:14:32.063256 3796 gateway.go:151] Starting gateway service sync I0514 23:14:32.063688 3796 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0514 23:14:32.066099 3796 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0514 23:14:32.092980 3796 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0514 23:14:32.094260 3796 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0514 23:14:32.095491 3796 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0514 23:14:32.129508 3796 gateway.go:168] Gateway service sync done. Time taken: 66.239248ms I0514 23:14:32.129925 3796 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32423 I0514 23:14:32.130269 3796 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000d1c150 0xc0002b7d50 0xc0002b7dc0 0xc0002b7e30 0xc0002b7ea0 0xc0002b7f10 0xc0002ca070 0xc000471d60 0xc0002ca0e0 0xc0002ca150 0xc000b822d0 0xc0061e7500 0xc0002ca1c0 map[0x2ff9080:0xc00249e120 0x2ff95c0:0xc00249e420 0x2ffa2e0:0xc00249e060 0x2ffa580:0xc00249e0c0 0x2ffad60:0xc00249e180 0x2ffb000:0xc00249e1e0 0x2ffb2a0:0xc00249e3c0 0x2ffba80:0xc00249e480 0x2ffbd20:0xc0068a4000 0x3026980:0xc001097200 0x3026d20:0xc001097260 0x3027800:0xc000d42660 0x3029160:0xc001bca000 0x30298a0:0xc00249e000 0x302a720:0xc001096de0] 0xc00047e540 0} I0514 23:14:32.130704 3796 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000d1c150 0xc0002b7d50 0xc0002b7dc0 0xc0002b7e30 0xc0002b7ea0 0xc0002b7f10 0xc0002ca070 0xc000471d60 0xc0002ca0e0 0xc0002ca150 0xc000b822d0 0xc0061e7500 0xc0002ca1c0 map[0x2ff9080:0xc00249e120 0x2ff95c0:0xc00249e420 0x2ffa2e0:0xc00249e060 0x2ffa580:0xc00249e0c0 0x2ffad60:0xc00249e180 0x2ffb000:0xc00249e1e0 0x2ffb2a0:0xc00249e3c0 0x2ffba80:0xc00249e480 0x2ffbd20:0xc0068a4000 0x3026980:0xc001097200 0x3026d20:0xc001097260 0x3027800:0xc000d42660 0x3029160:0xc001bca000 0x30298a0:0xc00249e000 0x302a720:0xc001096de0] 0xc00047e540 0} I0514 23:14:32.130741 3796 gateway.go:532] Reconciling gateway with updates I0514 23:14:32.136093 3796 gateway.go:339] Spawning Conntrack Rule Check Thread I0514 23:14:32.136181 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:32.136192 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:32.136207 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:32.136206 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:32.136215 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:32.136223 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:32.136251 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:32.136258 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:32.136271 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:32.136273 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0514 23:14:32.136278 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:32.136188 3796 default_node_network_controller.go:944] Gateway and management port readiness took 73.006032ms I0514 23:14:32.136283 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0514 23:14:32.136289 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0514 23:14:32.136299 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:32.136306 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:32.136313 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:32.136315 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0514 23:14:32.136317 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:32.136320 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:32.136324 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0514 23:14:32.136271 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:32.136338 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:32.136343 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:32.136344 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:32.136341 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:32.136348 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0514 23:14:32.136344 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:32.136352 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:32.136354 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:32.136361 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:32.136362 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0514 23:14:32.136366 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:32.136370 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:32.136371 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:32.136369 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:32.136382 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:32.136387 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:32.136383 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:32.136390 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0514 23:14:32.136392 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:32.136396 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:32.136398 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:32.136405 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:32.136405 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:32.136393 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:32.136329 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:32.136411 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0514 23:14:32.136422 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:32.136323 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:32.136426 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0514 23:14:32.136432 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:32.136409 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:32.136342 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:32.136437 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0514 23:14:32.136444 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:32.136444 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0514 23:14:32.136357 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:32.136452 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:32.136344 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:32.136460 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:32.136437 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0514 23:14:32.136307 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:32.136410 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0514 23:14:32.136448 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:32.136357 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0514 23:14:32.136438 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:32.136475 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:32.136510 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:32.136519 3796 port_claim.go:71] Svc openshift-ingress/router-default: port 31443 is already open, no action needed I0514 23:14:32.136529 3796 port_claim.go:71] Svc openshift-ingress/router-default: port 30875 is already open, no action needed I0514 23:14:32.136460 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:32.136543 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:32.136551 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:32.136556 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0514 23:14:32.136580 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:32.136594 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:32.136693 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:32.136701 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:32.136709 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:32.136718 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:32.136729 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0514 23:14:32.136738 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0514 23:14:32.136744 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:32.136755 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:32.136767 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:32.136770 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:32.136783 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:32.136758 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:32.136824 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:32.136830 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0514 23:14:32.136839 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0514 23:14:32.136841 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:32.136839 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:32.136844 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0514 23:14:32.136847 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0514 23:14:32.136857 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:32.136852 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0514 23:14:32.136862 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:32.136863 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:32.136868 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:32.136874 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:32.136877 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:32.136875 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:32.136886 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:32.136845 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:32.136900 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:32.136862 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0514 23:14:32.136854 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:32.136729 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0514 23:14:32.137788 3796 node_ip_handler_linux.go:160] Node IP manager is running W0514 23:14:32.176422 3796 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0514 23:14:32.176538 3796 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0514 23:14:32.176640 3796 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0514 23:14:32.184540 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0514 23:14:32.184562 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0514 23:14:32.184569 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0514 23:14:32.184575 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0514 23:14:32.184580 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0514 23:14:32.184586 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0514 23:14:32.184625 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0514 23:14:32.184624 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0514 23:14:32.184632 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0514 23:14:32.184639 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0514 23:14:32.224625 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0514 23:14:32.224630 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0514 23:14:32.224640 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0514 23:14:32.224646 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0514 23:14:32.224658 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0514 23:14:32.224664 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) W0514 23:14:32.258644 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0514 23:14:32.264164 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0514 23:14:32.264195 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0514 23:14:32.264202 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0514 23:14:32.264208 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0514 23:14:32.264214 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0514 23:14:32.264219 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0514 23:14:32.264301 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0514 23:14:32.264321 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0514 23:14:32.264628 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) W0514 23:14:32.438003 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:14:32.481797 3796 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:32.749388 3796 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0514 23:14:32.750231 3796 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 824.365µs I0514 23:14:32.803723 3796 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0514 23:14:33.098218 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:33.098240 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 28.356µs I0514 23:14:33.098242 3796 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:33.098315 3796 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:33.684731 3796 namespace.go:98] [openshift-service-ca] adding namespace I0514 23:14:33.685754 3796 namespace.go:102] [openshift-service-ca] adding namespace took 1.002563ms I0514 23:14:33.700981 3796 namespace.go:142] [openshift-service-ca] updating namespace I0514 23:14:34.071762 3796 default_node_network_controller.go:1380] Starting node controller node sync W0514 23:14:34.071781 3796 egressip_healthcheck.go:76] Health checking using insecure connection I0514 23:14:34.071835 3796 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0514 23:14:34.083724 3796 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.940646ms I0514 23:14:34.083838 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-98.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0514 23:14:34.083853 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-98.ec2.internal", IP: 10.0.1.98 to PMTUD blocking rules I0514 23:14:34.089953 3796 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0514 23:14:34.090242 3796 egressservice_node.go:93] Setting up event handlers for Egress Services I0514 23:14:34.090386 3796 egressservice_node.go:183] Starting Egress Services Controller I0514 23:14:34.090403 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0514 23:14:34.090419 3796 shared_informer.go:356] "Caches are synced" controller="egressservices" I0514 23:14:34.090428 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0514 23:14:34.090436 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0514 23:14:34.090446 3796 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0514 23:14:34.090455 3796 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0514 23:14:34.090459 3796 egressservice_node.go:197] Repairing Egress Services I0514 23:14:34.103287 3796 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0514 23:14:34.103299 3796 external_controller.go:278] Starting Admin Policy Based Route Controller I0514 23:14:34.103425 3796 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0514 23:14:34.103434 3796 link_network_manager.go:119] Link manager is running I0514 23:14:34.103442 3796 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0514 23:14:34.103772 3796 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0514 23:14:34.103807 3796 vrf_manager.go:128] VRF manager is running I0514 23:14:34.103895 3796 node_controller_manager.go:437] Removing flows to drop GARP I0514 23:14:34.103904 3796 gateway.go:532] Reconciling gateway with updates I0514 23:14:34.125519 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:34.125540 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:14:34.125538 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:34.125550 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:34.125550 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:34.125564 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0514 23:14:34.125565 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:34.125571 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0514 23:14:34.125569 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:34.125591 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0514 23:14:34.125589 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:34.125621 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0514 23:14:34.125630 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0514 23:14:34.125621 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:34.125631 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:34.125648 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0514 23:14:34.125654 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0514 23:14:34.125656 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0514 23:14:34.125654 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0514 23:14:34.125660 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:34.125655 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:34.125674 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0514 23:14:34.125675 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0514 23:14:34.125698 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0514 23:14:34.125700 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:34.125713 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:34.125706 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:34.125722 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0514 23:14:34.125711 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:34.125728 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:34.125730 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0514 23:14:34.125621 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:34.125698 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0514 23:14:34.125738 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:34.125555 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0514 23:14:34.125742 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:34.125749 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0514 23:14:34.125749 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:34.125749 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:34.125756 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0514 23:14:34.125758 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0514 23:14:34.125757 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:34.125765 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0514 23:14:34.125591 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:34.125775 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:34.125739 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:34.125785 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0514 23:14:34.125787 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:34.125541 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:34.125792 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0514 23:14:34.125797 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0514 23:14:34.125729 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:34.125801 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:34.125804 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0514 23:14:34.125572 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0514 23:14:34.125813 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0514 23:14:34.125822 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:34.125842 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0514 23:14:34.125786 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0514 23:14:34.125745 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0514 23:14:34.125577 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:34.125721 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:34.125914 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0514 23:14:34.125920 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0514 23:14:34.125922 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0514 23:14:34.125723 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:34.125923 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:34.125928 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:34.125946 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0514 23:14:34.125953 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0514 23:14:34.125969 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0514 23:14:34.125949 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:34.125989 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0514 23:14:34.125934 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0514 23:14:34.126003 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0514 23:14:34.125947 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0514 23:14:34.125731 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0514 23:14:34.125736 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0514 23:14:34.125738 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0514 23:14:34.125644 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:34.126169 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0514 23:14:34.126179 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0514 23:14:34.125579 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:34.126196 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0514 23:14:34.125742 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0514 23:14:34.125744 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:34.126238 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0514 23:14:34.126243 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0514 23:14:34.125756 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:34.126249 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0514 23:14:34.126252 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0514 23:14:34.125771 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0514 23:14:34.126273 3796 port_claim.go:71] Svc openshift-ingress/router-default: port 31443 is already open, no action needed I0514 23:14:34.126282 3796 port_claim.go:71] Svc openshift-ingress/router-default: port 30875 is already open, no action needed I0514 23:14:34.125782 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0514 23:14:34.125791 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0514 23:14:34.126338 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0514 23:14:34.125800 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0514 23:14:34.125701 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0514 23:14:34.125809 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0514 23:14:34.125726 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0514 23:14:34.125675 3796 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:34.126404 3796 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0514 23:14:34.150364 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0514 23:14:34.150388 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0514 23:14:34.150399 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0514 23:14:34.150406 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0514 23:14:34.150416 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0514 23:14:34.150432 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0514 23:14:34.150441 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0514 23:14:34.150449 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0514 23:14:34.150464 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0514 23:14:34.150481 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0514 23:14:34.150490 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0514 23:14:34.150504 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0514 23:14:34.150510 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0514 23:14:34.150515 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0514 23:14:34.150520 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0514 23:14:34.150525 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0514 23:14:34.150530 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0514 23:14:34.191994 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0514 23:14:34.192029 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0514 23:14:34.192040 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0514 23:14:34.192048 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0514 23:14:34.192071 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0514 23:14:34.192095 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0514 23:14:34.192103 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0514 23:14:34.192110 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0514 23:14:34.236520 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0514 23:14:34.236533 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0514 23:14:34.236552 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0514 23:14:34.236558 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0514 23:14:34.236565 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0514 23:14:34.236572 3796 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0514 23:14:35.627622 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-152.ec2.internal I0514 23:14:35.627655 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-152.ec2.internal: 56.414µs I0514 23:14:35.657884 3796 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-55bc57d65c-mbggh] creating logical port openshift-image-registry_image-registry-55bc57d65c-mbggh for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:35.658043 3796 kube.go:256] Updating pod openshift-image-registry/image-registry-55bc57d65c-mbggh I0514 23:14:35.664800 3796 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-778445446b-nzlw9] creating logical port openshift-ingress_router-default-778445446b-nzlw9 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:35.664937 3796 kube.go:256] Updating pod openshift-ingress/router-default-778445446b-nzlw9 I0514 23:14:35.679197 3796 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-xgjkc] creating logical port openshift-ingress-canary_ingress-canary-xgjkc for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:35.679308 3796 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-xgjkc I0514 23:14:35.681476 3796 pod.go:62] [openshift-image-registry/image-registry-55bc57d65c-mbggh] pod update took 23.466587ms I0514 23:14:35.681499 3796 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-55bc57d65c-mbggh] addLogicalPort annotation time took 23.503411ms I0514 23:14:35.682864 3796 pods.go:271] [openshift-image-registry/image-registry-55bc57d65c-mbggh] addLogicalPort took 25.003157ms, libovsdb time 1.021584ms I0514 23:14:35.688337 3796 pod.go:62] [openshift-ingress/router-default-778445446b-nzlw9] pod update took 23.443571ms I0514 23:14:35.688354 3796 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-778445446b-nzlw9] addLogicalPort annotation time took 23.464984ms I0514 23:14:35.689498 3796 pods.go:271] [openshift-ingress/router-default-778445446b-nzlw9] addLogicalPort took 24.712004ms, libovsdb time 901.981µs I0514 23:14:35.697877 3796 pod.go:62] [openshift-ingress-canary/ingress-canary-xgjkc] pod update took 18.596818ms I0514 23:14:35.697900 3796 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-xgjkc] addLogicalPort annotation time took 18.62242ms I0514 23:14:35.698173 3796 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-wdsmd] creating logical port openshift-dns_dns-default-wdsmd for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:35.698295 3796 kube.go:256] Updating pod openshift-dns/dns-default-wdsmd I0514 23:14:35.699149 3796 pods.go:271] [openshift-ingress-canary/ingress-canary-xgjkc] addLogicalPort took 19.967644ms, libovsdb time 953.144µs I0514 23:14:35.715937 3796 pod.go:62] [openshift-dns/dns-default-wdsmd] pod update took 17.67463ms I0514 23:14:35.715963 3796 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-wdsmd] addLogicalPort annotation time took 17.704125ms I0514 23:14:35.717202 3796 pods.go:271] [openshift-dns/dns-default-wdsmd] addLogicalPort took 19.038612ms, libovsdb time 918.99µs I0514 23:14:36.770234 3796 cni.go:452] [openshift-ingress-canary/ingress-canary-xgjkc 02312aa5d3a81c2051f85b49710d8b98325495a3bcf60e6b782724169133c11d network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-xgjkc 02312aa5d3a81c2051f85b49710d8b98325495a3bcf60e6b782724169133c11d network default NAD default NAD key ] I0514 23:14:36.812415 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-xgjkc, hostIfaceName: 02312aa5d3a81c2, network: default, NAD default, SandboxID: "02312aa5d3a81c2051f85b49710d8b98325495a3bcf60e6b782724169133c11d", PCI device ID: , UID: "f77f76e2-5b7d-4213-af31-779eabfd41c6", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0514 23:14:36.875468 3796 cni.go:473] [openshift-ingress-canary/ingress-canary-xgjkc 02312aa5d3a81c2051f85b49710d8b98325495a3bcf60e6b782724169133c11d network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-xgjkc 02312aa5d3a81c2051f85b49710d8b98325495a3bcf60e6b782724169133c11d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:1f:6b:70:a5:4a\",\"name\":\"02312aa5d3a81c2\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ff67cd99-6d12-430d-ba02-3dd426d5e5ec\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:14:39.044375 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:39.044413 3796 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "ip-10-0-0-78.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:39.044424 3796 node_tracker.go:235] Node ip-10-0-0-78.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-78.ec2.internal" I0514 23:14:39.044441 3796 node_tracker.go:169] Node ip-10-0-0-78.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:39.044455 3796 services_controller.go:574] Full service sync requested for network=default W0514 23:14:39.044822 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0514 23:14:39.044954 3796 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0514 23:14:39.045028 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0514 23:14:39.131868 3796 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "ip-10-0-0-78.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:39.166934 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:39.166940 3796 default_network_controller.go:1025] Node "ip-10-0-0-78.ec2.internal" in remote zone "ip-10-0-0-78.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0514 23:14:39.166991 3796 node_tracker.go:169] Node ip-10-0-0-78.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:39.167002 3796 services_controller.go:574] Full service sync requested for network=default W0514 23:14:39.167303 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0514 23:14:39.167389 3796 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0514 23:14:39.167430 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0514 23:14:39.167907 3796 zone_ic_handler.go:195] Time taken to create transit switch: 281.735µs I0514 23:14:39.167920 3796 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-78.ec2.internal for the network default I0514 23:14:39.168858 3796 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-78.ec2.internal" on network "default" took: 1.257728ms I0514 23:14:39.336497 3796 cni.go:452] [kube-system/global-pull-secret-syncer-p6k99 1235db229b5d6c097b03b3b35ab29ec752618b91b45618f53483cdd0f65eb37f network default NAD default NAD key ] ADD starting CNI request [kube-system/global-pull-secret-syncer-p6k99 1235db229b5d6c097b03b3b35ab29ec752618b91b45618f53483cdd0f65eb37f network default NAD default NAD key ] I0514 23:14:39.346689 3796 helper_linux.go:556] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-p6k99, hostIfaceName: 1235db229b5d6c0, network: default, NAD default, SandboxID: "1235db229b5d6c097b03b3b35ab29ec752618b91b45618f53483cdd0f65eb37f", PCI device ID: , UID: "4d83bde8-2190-45fc-ab07-75d72e866878", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0514 23:14:39.392587 3796 cni.go:473] [kube-system/global-pull-secret-syncer-p6k99 1235db229b5d6c097b03b3b35ab29ec752618b91b45618f53483cdd0f65eb37f network default NAD default NAD key default] ADD finished CNI request [kube-system/global-pull-secret-syncer-p6k99 1235db229b5d6c097b03b3b35ab29ec752618b91b45618f53483cdd0f65eb37f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:5c:c9:2d:a9:2b\",\"name\":\"1235db229b5d6c0\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8ea104b7-a32e-4954-97c2-29a1aad18891\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:14:39.443259 3796 cni.go:452] [openshift-network-diagnostics/network-check-target-snbfj 864bb98746c409a4ad55ac7de62115027d49cdac88cfc281c8aa4e16c328a70e network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-snbfj 864bb98746c409a4ad55ac7de62115027d49cdac88cfc281c8aa4e16c328a70e network default NAD default NAD key ] I0514 23:14:39.454640 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-snbfj, hostIfaceName: 864bb98746c409a, network: default, NAD default, SandboxID: "864bb98746c409a4ad55ac7de62115027d49cdac88cfc281c8aa4e16c328a70e", PCI device ID: , UID: "43442df5-7a54-4ef6-988f-fe95fcff3c88", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0514 23:14:39.500787 3796 cni.go:473] [openshift-network-diagnostics/network-check-target-snbfj 864bb98746c409a4ad55ac7de62115027d49cdac88cfc281c8aa4e16c328a70e network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-snbfj 864bb98746c409a4ad55ac7de62115027d49cdac88cfc281c8aa4e16c328a70e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:42:16:d4:3d:4b\",\"name\":\"864bb98746c409a\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bed46dce-4311-4dd5-b7d5-a13504c32624\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:14:40.465675 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0514 23:14:40.784856 3796 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-78.ec2.internal in network "default" I0514 23:14:40.784918 3796 egressfirewall.go:723] Syncing node "ip-10-0-0-78.ec2.internal" for egress firewall I0514 23:14:40.784934 3796 node_tracker.go:169] Node ip-10-0-0-78.ec2.internal switch + router changed, syncing services in network "default" I0514 23:14:40.784952 3796 services_controller.go:574] Full service sync requested for network=default I0514 23:14:40.784866 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:40.785026 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-78.ec2.internal", IP: 10.0.0.78 to PMTUD blocking rules I0514 23:14:40.785029 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 165.63µs I0514 23:14:40.785043 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-78.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0514 23:14:40.785055 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:14:40.785436 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0514 23:14:40.791708 3796 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-78.ec2.internal. Updating no re-route policies I0514 23:14:40.821244 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-78.ec2.internal", IP: 10.0.0.78 to PMTUD blocking rules I0514 23:14:40.821272 3796 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-78.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0514 23:14:41.087862 3796 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-854969bbd6-gtc27 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-854969bbd6-gtc27: no pod IPs found I0514 23:14:42.294622 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:14:42.295346 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:14:42.312240 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:14:42.312260 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-h99kj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:14:42.320772 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-55bc57d65c-h99kj I0514 23:14:42.320798 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-55bc57d65c-h99kj I0514 23:14:44.233296 3796 cni.go:452] [openshift-dns/dns-default-wdsmd 054afe83119298c9c9576c3df9c29698ee4a69f3f4adf38888153e21fdbe0dbf network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-wdsmd 054afe83119298c9c9576c3df9c29698ee4a69f3f4adf38888153e21fdbe0dbf network default NAD default NAD key ] I0514 23:14:44.233489 3796 cni.go:452] [openshift-image-registry/image-registry-55bc57d65c-mbggh 543bb54bdc182e3a41be6a013b7ab0bc51d386faebca3dd083a7162e7f421bc7 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-55bc57d65c-mbggh 543bb54bdc182e3a41be6a013b7ab0bc51d386faebca3dd083a7162e7f421bc7 network default NAD default NAD key ] I0514 23:14:44.244952 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-wdsmd, hostIfaceName: 054afe83119298c, network: default, NAD default, SandboxID: "054afe83119298c9c9576c3df9c29698ee4a69f3f4adf38888153e21fdbe0dbf", PCI device ID: , UID: "328eb4e4-84e4-46a6-89cd-930136756e58", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0514 23:14:44.244983 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-55bc57d65c-mbggh, hostIfaceName: 543bb54bdc182e3, network: default, NAD default, SandboxID: "543bb54bdc182e3a41be6a013b7ab0bc51d386faebca3dd083a7162e7f421bc7", PCI device ID: , UID: "42aaaa42-2a4f-453e-bc8c-6cbd809af067", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0514 23:14:44.297506 3796 cni.go:473] [openshift-dns/dns-default-wdsmd 054afe83119298c9c9576c3df9c29698ee4a69f3f4adf38888153e21fdbe0dbf network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-wdsmd 054afe83119298c9c9576c3df9c29698ee4a69f3f4adf38888153e21fdbe0dbf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:e3:06:27:ee:dc\",\"name\":\"054afe83119298c\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6baef629-89f3-4941-9f0d-33cc747edf01\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:14:44.313799 3796 cni.go:473] [openshift-image-registry/image-registry-55bc57d65c-mbggh 543bb54bdc182e3a41be6a013b7ab0bc51d386faebca3dd083a7162e7f421bc7 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-55bc57d65c-mbggh 543bb54bdc182e3a41be6a013b7ab0bc51d386faebca3dd083a7162e7f421bc7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:2a:e1:79:cf:32\",\"name\":\"543bb54bdc182e3\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7ca1e37d-3fcd-4751-b6f9-8216c0e7aa71\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:14:51.676708 3796 cni.go:452] [openshift-ingress/router-default-778445446b-nzlw9 d8e2d9dc6f300487cdcd886f1488b55049587b2970e3eaf597e2e87fc7921ed9 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-778445446b-nzlw9 d8e2d9dc6f300487cdcd886f1488b55049587b2970e3eaf597e2e87fc7921ed9 network default NAD default NAD key ] I0514 23:14:51.687075 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-778445446b-nzlw9, hostIfaceName: d8e2d9dc6f30048, network: default, NAD default, SandboxID: "d8e2d9dc6f300487cdcd886f1488b55049587b2970e3eaf597e2e87fc7921ed9", PCI device ID: , UID: "c31c8580-076e-4123-ab97-25862f40853c", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0514 23:14:51.734385 3796 cni.go:473] [openshift-ingress/router-default-778445446b-nzlw9 d8e2d9dc6f300487cdcd886f1488b55049587b2970e3eaf597e2e87fc7921ed9 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-778445446b-nzlw9 d8e2d9dc6f300487cdcd886f1488b55049587b2970e3eaf597e2e87fc7921ed9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:b2:e4:3b:7a:10\",\"name\":\"d8e2d9dc6f30048\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/08d61173-87ca-4360-9bac-616c1d26d4d2\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:14:57.946839 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:14:58.006348 3796 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0514 23:14:58.060723 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5g7gm] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-5g7gm for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:58.060839 3796 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5g7gm I0514 23:14:58.076897 3796 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5g7gm] pod update took 16.092134ms I0514 23:14:58.076922 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5g7gm] addLogicalPort annotation time took 16.121033ms I0514 23:14:58.078042 3796 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5g7gm] addLogicalPort took 17.336351ms, libovsdb time 824.224µs I0514 23:14:58.090561 3796 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0514 23:14:58.093351 3796 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:14:58.142267 3796 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-d477cf66b-rhxtq IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-d477cf66b-rhxtq: no pod IPs found I0514 23:14:58.359522 3796 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-ghbkm] creating logical port openshift-console_downloads-5c54f4944f-ghbkm for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:58.359668 3796 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-ghbkm I0514 23:14:58.375046 3796 pod.go:62] [openshift-console/downloads-5c54f4944f-ghbkm] pod update took 15.400631ms I0514 23:14:58.375072 3796 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-ghbkm] addLogicalPort annotation time took 15.432302ms I0514 23:14:58.376222 3796 pods.go:271] [openshift-console/downloads-5c54f4944f-ghbkm] addLogicalPort took 16.713123ms, libovsdb time 835.314µs I0514 23:14:58.470490 3796 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5g7gm 8787f221bbd8737982d6233009d4e755a45195bafe84c5355982d051ebe3c913 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5g7gm 8787f221bbd8737982d6233009d4e755a45195bafe84c5355982d051ebe3c913 network default NAD default NAD key ] I0514 23:14:58.480687 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-5g7gm, hostIfaceName: 8787f221bbd8737, network: default, NAD default, SandboxID: "8787f221bbd8737982d6233009d4e755a45195bafe84c5355982d051ebe3c913", PCI device ID: , UID: "218c888c-00ee-4495-bc7f-e3a173f2c08e", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0514 23:14:58.513875 3796 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-6nk4g] creating logical port openshift-insights_insights-runtime-extractor-6nk4g for pod on switch ip-10-0-2-152.ec2.internal I0514 23:14:58.513998 3796 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-6nk4g I0514 23:14:58.530362 3796 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5g7gm 8787f221bbd8737982d6233009d4e755a45195bafe84c5355982d051ebe3c913 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5g7gm 8787f221bbd8737982d6233009d4e755a45195bafe84c5355982d051ebe3c913 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:d6:57:be:31:f9\",\"name\":\"8787f221bbd8737\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b4709195-4d87-4cf8-8383-1e28e2650266\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:14:58.532052 3796 pod.go:62] [openshift-insights/insights-runtime-extractor-6nk4g] pod update took 18.105303ms I0514 23:14:58.532072 3796 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-6nk4g] addLogicalPort annotation time took 18.130329ms I0514 23:14:58.532264 3796 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-78.ec2.internal I0514 23:14:58.532299 3796 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-78.ec2.internal: 38.334µs I0514 23:14:58.533271 3796 pods.go:271] [openshift-insights/insights-runtime-extractor-6nk4g] addLogicalPort took 19.437149ms, libovsdb time 823.628µs I0514 23:14:58.795005 3796 cni.go:452] [openshift-console/downloads-5c54f4944f-ghbkm d503349d3b9ce83c09be35295c064139aeedb84ddc8c4376043c27beefd9942d network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-ghbkm d503349d3b9ce83c09be35295c064139aeedb84ddc8c4376043c27beefd9942d network default NAD default NAD key ] I0514 23:14:58.804909 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-ghbkm, hostIfaceName: d503349d3b9ce83, network: default, NAD default, SandboxID: "d503349d3b9ce83c09be35295c064139aeedb84ddc8c4376043c27beefd9942d", PCI device ID: , UID: "1270dd92-23ac-462c-8537-fc45abbfbda6", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0514 23:14:58.851537 3796 cni.go:473] [openshift-console/downloads-5c54f4944f-ghbkm d503349d3b9ce83c09be35295c064139aeedb84ddc8c4376043c27beefd9942d network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-ghbkm d503349d3b9ce83c09be35295c064139aeedb84ddc8c4376043c27beefd9942d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:45:7a:d7:65:14\",\"name\":\"d503349d3b9ce83\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/038e97f4-5d11-4a94-bddf-c1c13850e4a0\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:14:58.915925 3796 cni.go:452] [openshift-insights/insights-runtime-extractor-6nk4g 61f045247e02a9e93b28eb9932cc732473d8031303b67824c1083cbe02ef2a85 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-6nk4g 61f045247e02a9e93b28eb9932cc732473d8031303b67824c1083cbe02ef2a85 network default NAD default NAD key ] I0514 23:14:58.925687 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-6nk4g, hostIfaceName: 61f045247e02a9e, network: default, NAD default, SandboxID: "61f045247e02a9e93b28eb9932cc732473d8031303b67824c1083cbe02ef2a85", PCI device ID: , UID: "f70453fb-eb6c-4fd4-9e68-856dac713a6b", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0514 23:14:58.971808 3796 cni.go:473] [openshift-insights/insights-runtime-extractor-6nk4g 61f045247e02a9e93b28eb9932cc732473d8031303b67824c1083cbe02ef2a85 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-6nk4g 61f045247e02a9e93b28eb9932cc732473d8031303b67824c1083cbe02ef2a85 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:3f:53:c3:19:77\",\"name\":\"61f045247e02a9e\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/de87f6ea-d087-450d-ab29-cd7aef47097b\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:15:00.276490 3796 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:15:00.515589 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0514 23:15:00.542808 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0514 23:15:02.062150 3796 base_network_controller_pods.go:487] [default/openshift-console/console-b57f75797-44prb] creating logical port openshift-console_console-b57f75797-44prb for pod on switch ip-10-0-2-152.ec2.internal I0514 23:15:02.062272 3796 kube.go:256] Updating pod openshift-console/console-b57f75797-44prb I0514 23:15:02.087200 3796 pod.go:62] [openshift-console/console-b57f75797-44prb] pod update took 24.961373ms I0514 23:15:02.087222 3796 base_network_controller_pods.go:951] [default/openshift-console/console-b57f75797-44prb] addLogicalPort annotation time took 24.988957ms I0514 23:15:02.088391 3796 pods.go:271] [openshift-console/console-b57f75797-44prb] addLogicalPort took 26.258881ms, libovsdb time 856.038µs I0514 23:15:02.423284 3796 cni.go:452] [openshift-console/console-b57f75797-44prb 37cbecbb1e4f420e953a80b81d9580fd1c57fcfaa4d427cb579d2c610f88b9ae network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-b57f75797-44prb 37cbecbb1e4f420e953a80b81d9580fd1c57fcfaa4d427cb579d2c610f88b9ae network default NAD default NAD key ] I0514 23:15:02.434909 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-b57f75797-44prb, hostIfaceName: 37cbecbb1e4f420, network: default, NAD default, SandboxID: "37cbecbb1e4f420e953a80b81d9580fd1c57fcfaa4d427cb579d2c610f88b9ae", PCI device ID: , UID: "b4c08eca-951f-4ccc-a0b1-4a4cebb2f911", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0514 23:15:02.489330 3796 cni.go:473] [openshift-console/console-b57f75797-44prb 37cbecbb1e4f420e953a80b81d9580fd1c57fcfaa4d427cb579d2c610f88b9ae network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-b57f75797-44prb 37cbecbb1e4f420e953a80b81d9580fd1c57fcfaa4d427cb579d2c610f88b9ae network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:e3:a6:56:3a:65\",\"name\":\"37cbecbb1e4f420\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d8c0e94a-b007-458e-8c39-9b8150986905\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:15:02.525587 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-qrbkk] creating logical port openshift-monitoring_prometheus-operator-76b589d668-qrbkk for pod on switch ip-10-0-2-152.ec2.internal I0514 23:15:02.525722 3796 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-qrbkk I0514 23:15:02.546355 3796 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-qrbkk] pod update took 20.654951ms I0514 23:15:02.546381 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-qrbkk] addLogicalPort annotation time took 20.687421ms I0514 23:15:02.547595 3796 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-qrbkk] addLogicalPort took 22.02567ms, libovsdb time 838.814µs W0514 23:15:03.261950 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0514 23:15:03.261977 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0514 23:15:03.493846 3796 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-qrbkk d40ce67258dfc092b1c4ad692fa32f23553f80c8e72fb33cf0f3b5924b6fef75 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-qrbkk d40ce67258dfc092b1c4ad692fa32f23553f80c8e72fb33cf0f3b5924b6fef75 network default NAD default NAD key ] I0514 23:15:03.505154 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-qrbkk, hostIfaceName: d40ce67258dfc09, network: default, NAD default, SandboxID: "d40ce67258dfc092b1c4ad692fa32f23553f80c8e72fb33cf0f3b5924b6fef75", PCI device ID: , UID: "fb512058-2135-4afc-9b4f-3ed4d68ef83d", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0514 23:15:03.558754 3796 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-qrbkk d40ce67258dfc092b1c4ad692fa32f23553f80c8e72fb33cf0f3b5924b6fef75 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-qrbkk d40ce67258dfc092b1c4ad692fa32f23553f80c8e72fb33cf0f3b5924b6fef75 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:5c:fd:17:06:e0\",\"name\":\"d40ce67258dfc09\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7a08665f-4404-417f-aa7b-bf175f876c39\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:15:04.316359 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:15:04.316462 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0514 23:15:06.294847 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0514 23:15:06.544536 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0514 23:15:09.001450 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-b8587444b-2vxt2] creating logical port openshift-monitoring_openshift-state-metrics-b8587444b-2vxt2 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:15:09.001577 3796 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-b8587444b-2vxt2 I0514 23:15:09.022266 3796 pod.go:62] [openshift-monitoring/openshift-state-metrics-b8587444b-2vxt2] pod update took 20.730062ms I0514 23:15:09.022293 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/openshift-state-metrics-b8587444b-2vxt2] addLogicalPort annotation time took 20.762461ms I0514 23:15:09.023512 3796 pods.go:271] [openshift-monitoring/openshift-state-metrics-b8587444b-2vxt2] addLogicalPort took 22.08123ms, libovsdb time 909.531µs I0514 23:15:10.269520 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:15:10.269719 3796 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0514 23:15:10.316272 3796 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 46.602388ms I0514 23:15:10.316299 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 46.6334ms I0514 23:15:10.317505 3796 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 48.005174ms, libovsdb time 823.386µs W0514 23:15:11.058710 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0514 23:15:12.138074 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-64c6d5bccf-fvkst] creating logical port openshift-monitoring_thanos-querier-64c6d5bccf-fvkst for pod on switch ip-10-0-2-152.ec2.internal I0514 23:15:12.138194 3796 kube.go:256] Updating pod openshift-monitoring/thanos-querier-64c6d5bccf-fvkst I0514 23:15:12.163934 3796 pod.go:62] [openshift-monitoring/thanos-querier-64c6d5bccf-fvkst] pod update took 25.784647ms I0514 23:15:12.163958 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-64c6d5bccf-fvkst] addLogicalPort annotation time took 25.816026ms I0514 23:15:12.165143 3796 pods.go:271] [openshift-monitoring/thanos-querier-64c6d5bccf-fvkst] addLogicalPort took 27.082183ms, libovsdb time 843.63µs W0514 23:15:12.582166 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0514 23:15:13.391297 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:13.391319 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:13.391326 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0514 23:15:13.691985 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-787cb4645c-sjbjf] creating logical port openshift-monitoring_metrics-server-787cb4645c-sjbjf for pod on switch ip-10-0-2-152.ec2.internal I0514 23:15:13.692154 3796 kube.go:256] Updating pod openshift-monitoring/metrics-server-787cb4645c-sjbjf I0514 23:15:13.721926 3796 pod.go:62] [openshift-monitoring/metrics-server-787cb4645c-sjbjf] pod update took 29.801268ms I0514 23:15:13.721946 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-787cb4645c-sjbjf] addLogicalPort annotation time took 29.827153ms I0514 23:15:13.723169 3796 pods.go:271] [openshift-monitoring/metrics-server-787cb4645c-sjbjf] addLogicalPort took 31.201771ms, libovsdb time 890.289µs I0514 23:15:13.790468 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-cstf4] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-cstf4 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:15:13.790563 3796 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-cstf4 I0514 23:15:13.808100 3796 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-cstf4] pod update took 17.575198ms I0514 23:15:13.808125 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-cstf4] addLogicalPort annotation time took 17.600873ms I0514 23:15:13.809321 3796 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-cstf4] addLogicalPort took 18.870236ms, libovsdb time 844.911µs I0514 23:15:13.899541 3796 cni.go:452] [openshift-multus/network-metrics-daemon-j7h6h c2713557da795aa09ce8500fc5c10674bee752aa6482aaef216780f7e2ffb484 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-j7h6h c2713557da795aa09ce8500fc5c10674bee752aa6482aaef216780f7e2ffb484 network default NAD default NAD key ] I0514 23:15:13.901700 3796 cni.go:452] [openshift-monitoring/alertmanager-main-1 71b9e3cadfbf50e8498be1796538fbede5f179cb2fff7ca084f63a9213e40234 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 71b9e3cadfbf50e8498be1796538fbede5f179cb2fff7ca084f63a9213e40234 network default NAD default NAD key ] I0514 23:15:13.903643 3796 cni.go:452] [openshift-monitoring/openshift-state-metrics-b8587444b-2vxt2 f65750aa1940c597fa0d108f710f5fdb007d56aceb46b3577012c2324302b95a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-2vxt2 f65750aa1940c597fa0d108f710f5fdb007d56aceb46b3577012c2324302b95a network default NAD default NAD key ] I0514 23:15:13.906129 3796 cni.go:452] [openshift-monitoring/thanos-querier-64c6d5bccf-fvkst 2f809110a955355a195eb6ee7d19d8baba18664ae3f1599f144c954743bfaa58 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-64c6d5bccf-fvkst 2f809110a955355a195eb6ee7d19d8baba18664ae3f1599f144c954743bfaa58 network default NAD default NAD key ] I0514 23:15:13.917522 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-j7h6h, hostIfaceName: c2713557da795aa, network: default, NAD default, SandboxID: "c2713557da795aa09ce8500fc5c10674bee752aa6482aaef216780f7e2ffb484", PCI device ID: , UID: "92a08bef-8664-4f88-b5c7-488a544ff444", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0514 23:15:13.917597 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 71b9e3cadfbf50e, network: default, NAD default, SandboxID: "71b9e3cadfbf50e8498be1796538fbede5f179cb2fff7ca084f63a9213e40234", PCI device ID: , UID: "ea14534c-9bf8-4c4e-81f0-369328aee4ef", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0514 23:15:13.918299 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-b8587444b-2vxt2, hostIfaceName: f65750aa1940c59, network: default, NAD default, SandboxID: "f65750aa1940c597fa0d108f710f5fdb007d56aceb46b3577012c2324302b95a", PCI device ID: , UID: "9d0f90c7-eecc-47f8-8232-179302630b26", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0514 23:15:13.930972 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-64c6d5bccf-fvkst, hostIfaceName: 2f809110a955355, network: default, NAD default, SandboxID: "2f809110a955355a195eb6ee7d19d8baba18664ae3f1599f144c954743bfaa58", PCI device ID: , UID: "c9c378ac-d8e1-4c49-96f5-8d7071afb7b4", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0514 23:15:13.974465 3796 cni.go:473] [openshift-multus/network-metrics-daemon-j7h6h c2713557da795aa09ce8500fc5c10674bee752aa6482aaef216780f7e2ffb484 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-j7h6h c2713557da795aa09ce8500fc5c10674bee752aa6482aaef216780f7e2ffb484 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:ad:de:c3:6e:e7\",\"name\":\"c2713557da795aa\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e2ae3686-d9bc-4fe4-93d7-8fde8e345863\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:15:14.040053 3796 cni.go:452] [openshift-monitoring/metrics-server-787cb4645c-sjbjf f75149bff02b4770f8d4711d8a676b075e4af62500b464b555513b276d5e448b network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-787cb4645c-sjbjf f75149bff02b4770f8d4711d8a676b075e4af62500b464b555513b276d5e448b network default NAD default NAD key ] I0514 23:15:14.057057 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-787cb4645c-sjbjf, hostIfaceName: f75149bff02b477, network: default, NAD default, SandboxID: "f75149bff02b4770f8d4711d8a676b075e4af62500b464b555513b276d5e448b", PCI device ID: , UID: "b2c70a0c-50d8-4af4-9f9b-8e51d6b11d89", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0514 23:15:14.111694 3796 cni.go:473] [openshift-monitoring/metrics-server-787cb4645c-sjbjf f75149bff02b4770f8d4711d8a676b075e4af62500b464b555513b276d5e448b network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-787cb4645c-sjbjf f75149bff02b4770f8d4711d8a676b075e4af62500b464b555513b276d5e448b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:70:64:47:0c:78\",\"name\":\"f75149bff02b477\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4c2b7876-f203-4058-9412-92e83ba7bf8c\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:15:14.148114 3796 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-cstf4 d90be6448383bfdf7aeafbe647cd1ea0c5896f7488c0ca8305a831b33bf8764a network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-cstf4 d90be6448383bfdf7aeafbe647cd1ea0c5896f7488c0ca8305a831b33bf8764a network default NAD default NAD key ] I0514 23:15:14.159038 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-cstf4, hostIfaceName: d90be6448383bfd, network: default, NAD default, SandboxID: "d90be6448383bfdf7aeafbe647cd1ea0c5896f7488c0ca8305a831b33bf8764a", PCI device ID: , UID: "739b1004-bd75-410b-8c3f-127e76fefbe4", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0514 23:15:14.198835 3796 cni.go:473] [openshift-monitoring/openshift-state-metrics-b8587444b-2vxt2 f65750aa1940c597fa0d108f710f5fdb007d56aceb46b3577012c2324302b95a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-b8587444b-2vxt2 f65750aa1940c597fa0d108f710f5fdb007d56aceb46b3577012c2324302b95a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:ac:a3:e4:0a:04\",\"name\":\"f65750aa1940c59\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/84bc37ee-0b7d-4f2d-8be6-bfa4ccb1214a\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:15:14.199399 3796 cni.go:473] [openshift-monitoring/thanos-querier-64c6d5bccf-fvkst 2f809110a955355a195eb6ee7d19d8baba18664ae3f1599f144c954743bfaa58 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-64c6d5bccf-fvkst 2f809110a955355a195eb6ee7d19d8baba18664ae3f1599f144c954743bfaa58 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:03:80:ec:f2:51\",\"name\":\"2f809110a955355\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e60265d0-68ed-4653-9d58-286c22ea1fde\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:15:14.202136 3796 cni.go:473] [openshift-monitoring/alertmanager-main-1 71b9e3cadfbf50e8498be1796538fbede5f179cb2fff7ca084f63a9213e40234 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 71b9e3cadfbf50e8498be1796538fbede5f179cb2fff7ca084f63a9213e40234 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:ca:68:d7:a0:25\",\"name\":\"71b9e3cadfbf50e\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c119787d-4990-4ba6-aee8-f5b326ee5dc0\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:15:14.211718 3796 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-cstf4 d90be6448383bfdf7aeafbe647cd1ea0c5896f7488c0ca8305a831b33bf8764a network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-cstf4 d90be6448383bfdf7aeafbe647cd1ea0c5896f7488c0ca8305a831b33bf8764a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:29:eb:f8:0a:e8\",\"name\":\"d90be6448383bfd\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ef51603d-a81b-4cbc-8762-e3a70260653d\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:15:14.454929 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0514 23:15:14.573997 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0514 23:15:14.600649 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0514 23:15:15.462841 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0514 23:15:15.540014 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:15:15.540187 3796 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0514 23:15:15.578297 3796 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 38.165143ms I0514 23:15:15.578330 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 38.205154ms I0514 23:15:15.579729 3796 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 39.73977ms, libovsdb time 1.006862ms W0514 23:15:16.271248 3796 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-79bf45776b-fn62n IP address from the namespace address-set, err: pod openshift-console/console-79bf45776b-fn62n: no pod IPs found W0514 23:15:16.378202 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:16.378221 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:18.414849 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:18.480324 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:18.480344 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:18.480350 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:18.480354 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0514 23:15:18.729842 3796 cni.go:452] [openshift-monitoring/prometheus-k8s-1 83c18c6bfc2eba1f424250134f0c57930fd0f98de3d0593aedaf88fc262f7ad1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 83c18c6bfc2eba1f424250134f0c57930fd0f98de3d0593aedaf88fc262f7ad1 network default NAD default NAD key ] I0514 23:15:18.741895 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 83c18c6bfc2eba1, network: default, NAD default, SandboxID: "83c18c6bfc2eba1f424250134f0c57930fd0f98de3d0593aedaf88fc262f7ad1", PCI device ID: , UID: "a6fb7dc7-98ef-47ef-8a7a-320113b7d939", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0514 23:15:18.804342 3796 cni.go:473] [openshift-monitoring/prometheus-k8s-1 83c18c6bfc2eba1f424250134f0c57930fd0f98de3d0593aedaf88fc262f7ad1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 83c18c6bfc2eba1f424250134f0c57930fd0f98de3d0593aedaf88fc262f7ad1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:bb:a3:a7:37:df\",\"name\":\"83c18c6bfc2eba1\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d9da8abe-c32b-48fa-8079-3658204b7972\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:15:19.644943 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:19.722078 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:19.722094 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:15:19.870206 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:19.870223 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:19.870228 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:21.650495 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:21.650614 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:21.650624 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:21.650631 3796 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0514 23:15:23.274212 3796 cni.go:452] [openshift-image-registry/image-registry-55bc57d65c-mbggh 543bb54bdc182e3a41be6a013b7ab0bc51d386faebca3dd083a7162e7f421bc7 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-55bc57d65c-mbggh 543bb54bdc182e3a41be6a013b7ab0bc51d386faebca3dd083a7162e7f421bc7 network default NAD default NAD key ] I0514 23:15:23.364953 3796 cni.go:473] [openshift-image-registry/image-registry-55bc57d65c-mbggh 543bb54bdc182e3a41be6a013b7ab0bc51d386faebca3dd083a7162e7f421bc7 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-55bc57d65c-mbggh 543bb54bdc182e3a41be6a013b7ab0bc51d386faebca3dd083a7162e7f421bc7 network default NAD default NAD key default], result "{}", err I0514 23:15:23.412463 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:23.412522 3796 pods.go:176] Deleting pod: openshift-image-registry/image-registry-55bc57d65c-mbggh I0514 23:15:23.413921 3796 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-55bc57d65c-mbggh, ips: 10.129.0.6 I0514 23:15:23.413951 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:23.424389 3796 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7fdc74bbd-7dxbb] creating logical port openshift-image-registry_image-registry-7fdc74bbd-7dxbb for pod on switch ip-10-0-2-152.ec2.internal I0514 23:15:23.424515 3796 kube.go:256] Updating pod openshift-image-registry/image-registry-7fdc74bbd-7dxbb I0514 23:15:23.444056 3796 pod.go:62] [openshift-image-registry/image-registry-7fdc74bbd-7dxbb] pod update took 19.56895ms I0514 23:15:23.444081 3796 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7fdc74bbd-7dxbb] addLogicalPort annotation time took 19.599253ms I0514 23:15:23.445261 3796 pods.go:271] [openshift-image-registry/image-registry-7fdc74bbd-7dxbb] addLogicalPort took 20.886522ms, libovsdb time 826.98µs I0514 23:15:23.692704 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:23.692730 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-55bc57d65c-mbggh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:23.695970 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-55bc57d65c-mbggh I0514 23:15:23.695991 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-55bc57d65c-mbggh I0514 23:15:23.783827 3796 cni.go:452] [openshift-image-registry/image-registry-7fdc74bbd-7dxbb 66b5c1ea3a533d98536d28e9c5bb12965652d915c55aaa42a83d67f4aa19728c network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7fdc74bbd-7dxbb 66b5c1ea3a533d98536d28e9c5bb12965652d915c55aaa42a83d67f4aa19728c network default NAD default NAD key ] I0514 23:15:23.793011 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7fdc74bbd-7dxbb, hostIfaceName: 66b5c1ea3a533d9, network: default, NAD default, SandboxID: "66b5c1ea3a533d98536d28e9c5bb12965652d915c55aaa42a83d67f4aa19728c", PCI device ID: , UID: "d534863d-0790-4f7b-9a18-aa64fff5e99e", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0514 23:15:23.843475 3796 cni.go:473] [openshift-image-registry/image-registry-7fdc74bbd-7dxbb 66b5c1ea3a533d98536d28e9c5bb12965652d915c55aaa42a83d67f4aa19728c network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7fdc74bbd-7dxbb 66b5c1ea3a533d98536d28e9c5bb12965652d915c55aaa42a83d67f4aa19728c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:78:b2:67:17:e3\",\"name\":\"66b5c1ea3a533d9\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/83bb3243-0fb1-4ca5-8312-fd119a10135d\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:15:24.400085 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.400104 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.400109 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.400115 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.400135 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.400141 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.400145 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:24.400149 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:28.683874 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:28.683897 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:28.683905 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0514 23:15:28.683911 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier I0514 23:15:36.385528 3796 obj_retry.go:498] Detected object openshift-console/console-b57f75797-f2pc2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:36.386075 3796 obj_retry.go:498] Detected object openshift-console/console-b57f75797-f2pc2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:36.779926 3796 obj_retry.go:492] Detected object openshift-console/console-b57f75797-f2pc2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:36.779949 3796 obj_retry.go:492] Detected object openshift-console/console-b57f75797-f2pc2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:36.788076 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-b57f75797-f2pc2 I0514 23:15:36.788089 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-b57f75797-f2pc2 W0514 23:15:37.574397 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0514 23:15:39.385735 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:39.385759 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:40.641235 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0514 23:15:40.670308 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:40.670329 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:40.670335 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:40.670348 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:40.670352 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:40.670358 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0514 23:15:42.909990 3796 obj_retry.go:498] Detected object openshift-console/console-79bf45776b-h4hm6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:42.910555 3796 obj_retry.go:498] Detected object openshift-console/console-79bf45776b-h4hm6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:43.641086 3796 obj_retry.go:492] Detected object openshift-console/console-79bf45776b-h4hm6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:43.641111 3796 obj_retry.go:492] Detected object openshift-console/console-79bf45776b-h4hm6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:43.648242 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-79bf45776b-h4hm6 I0514 23:15:43.648267 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-79bf45776b-h4hm6 W0514 23:15:43.678560 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0514 23:15:44.777369 3796 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.777940 3796 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.777970 3796 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.777978 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0514 23:15:44.777991 3796 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:15:44.777996 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... W0514 23:15:45.691296 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0514 23:15:45.815666 3796 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:45.815692 3796 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:45.815708 3796 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:45.815718 3796 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dpjbw7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:46.690570 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:46.691166 3796 obj_retry.go:498] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:15:47.628354 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:47.628378 3796 obj_retry.go:492] Detected object openshift-image-registry/image-registry-d477cf66b-ddgmf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:15:47.634098 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-d477cf66b-ddgmf I0514 23:15:47.634128 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-d477cf66b-ddgmf I0514 23:15:49.152319 3796 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0514 23:15:49.163142 3796 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0514 23:15:50.687345 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:50.687363 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:15:50.687369 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0514 23:15:54.312599 3796 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0514 23:16:02.710649 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0514 23:16:08.400953 3796 obj_retry.go:498] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:08.401474 3796 obj_retry.go:498] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:08.731388 3796 obj_retry.go:492] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:08.731414 3796 obj_retry.go:492] Detected object openshift-ingress/router-default-7f4c9f6df5-xjsx9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:08.735747 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7f4c9f6df5-xjsx9 I0514 23:16:08.735773 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7f4c9f6df5-xjsx9 I0514 23:16:12.736451 3796 cni.go:452] [openshift-console/console-b57f75797-44prb 37cbecbb1e4f420e953a80b81d9580fd1c57fcfaa4d427cb579d2c610f88b9ae network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-b57f75797-44prb 37cbecbb1e4f420e953a80b81d9580fd1c57fcfaa4d427cb579d2c610f88b9ae network default NAD default NAD key ] I0514 23:16:12.830859 3796 cni.go:473] [openshift-console/console-b57f75797-44prb 37cbecbb1e4f420e953a80b81d9580fd1c57fcfaa4d427cb579d2c610f88b9ae network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-b57f75797-44prb 37cbecbb1e4f420e953a80b81d9580fd1c57fcfaa4d427cb579d2c610f88b9ae network default NAD default NAD key default], result "{}", err I0514 23:16:12.876309 3796 obj_retry.go:498] Detected object openshift-console/console-b57f75797-44prb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:12.876346 3796 pods.go:176] Deleting pod: openshift-console/console-b57f75797-44prb I0514 23:16:12.877717 3796 pods.go:236] Attempting to release IPs for pod: openshift-console/console-b57f75797-44prb, ips: 10.129.0.13 I0514 23:16:12.877749 3796 obj_retry.go:498] Detected object openshift-console/console-b57f75797-44prb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:13.865270 3796 obj_retry.go:492] Detected object openshift-console/console-b57f75797-44prb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:13.865299 3796 obj_retry.go:492] Detected object openshift-console/console-b57f75797-44prb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:13.873754 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-b57f75797-44prb I0514 23:16:13.873785 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-b57f75797-44prb W0514 23:16:16.661679 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:16:16.661702 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:16:16.661717 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:16:16.661724 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:16:16.929108 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:16:16.929134 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0514 23:16:30.827346 3796 cni.go:452] [openshift-monitoring/alertmanager-main-1 71b9e3cadfbf50e8498be1796538fbede5f179cb2fff7ca084f63a9213e40234 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 71b9e3cadfbf50e8498be1796538fbede5f179cb2fff7ca084f63a9213e40234 network default NAD default NAD key ] I0514 23:16:30.911133 3796 cni.go:473] [openshift-monitoring/alertmanager-main-1 71b9e3cadfbf50e8498be1796538fbede5f179cb2fff7ca084f63a9213e40234 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 71b9e3cadfbf50e8498be1796538fbede5f179cb2fff7ca084f63a9213e40234 network default NAD default NAD key default], result "{}", err I0514 23:16:30.971489 3796 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:30.971525 3796 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0514 23:16:30.973025 3796 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.129.0.16 I0514 23:16:30.973059 3796 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:31.958339 3796 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:31.958371 3796 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:31.966180 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0514 23:16:31.966201 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0514 23:16:32.011726 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:16:32.011860 3796 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0514 23:16:32.051868 3796 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 40.046239ms I0514 23:16:32.051892 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 40.075434ms I0514 23:16:32.053109 3796 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 41.403658ms, libovsdb time 867.477µs I0514 23:16:32.372936 3796 cni.go:452] [openshift-monitoring/alertmanager-main-1 391788b5bb37f04ebdde6736296983a9fb1d0d1779b408fbdde1a3982dad8740 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 391788b5bb37f04ebdde6736296983a9fb1d0d1779b408fbdde1a3982dad8740 network default NAD default NAD key ] I0514 23:16:32.382348 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 391788b5bb37f04, network: default, NAD default, SandboxID: "391788b5bb37f04ebdde6736296983a9fb1d0d1779b408fbdde1a3982dad8740", PCI device ID: , UID: "2557a02e-0463-43d3-8166-8b733db4d6b8", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0514 23:16:32.432937 3796 cni.go:473] [openshift-monitoring/alertmanager-main-1 391788b5bb37f04ebdde6736296983a9fb1d0d1779b408fbdde1a3982dad8740 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 391788b5bb37f04ebdde6736296983a9fb1d0d1779b408fbdde1a3982dad8740 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:59:a3:d3:f0:ee\",\"name\":\"391788b5bb37f04\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f082c9e9-79c9-48c2-8b07-1775e6a13884\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:16:35.243032 3796 cni.go:452] [openshift-monitoring/prometheus-k8s-1 83c18c6bfc2eba1f424250134f0c57930fd0f98de3d0593aedaf88fc262f7ad1 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 83c18c6bfc2eba1f424250134f0c57930fd0f98de3d0593aedaf88fc262f7ad1 network default NAD default NAD key ] I0514 23:16:35.331219 3796 cni.go:473] [openshift-monitoring/prometheus-k8s-1 83c18c6bfc2eba1f424250134f0c57930fd0f98de3d0593aedaf88fc262f7ad1 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 83c18c6bfc2eba1f424250134f0c57930fd0f98de3d0593aedaf88fc262f7ad1 network default NAD default NAD key default], result "{}", err I0514 23:16:35.395760 3796 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:35.395793 3796 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0514 23:16:35.397197 3796 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.129.0.20 I0514 23:16:35.397232 3796 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:35.988023 3796 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:35.988053 3796 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:35.996680 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0514 23:16:35.996717 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0514 23:16:36.036625 3796 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:16:36.036733 3796 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0514 23:16:36.072399 3796 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 35.706524ms I0514 23:16:36.072426 3796 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 35.73804ms I0514 23:16:36.073723 3796 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 37.13832ms, libovsdb time 963.693µs I0514 23:16:36.397257 3796 cni.go:452] [openshift-monitoring/prometheus-k8s-1 5adb3f311fae3e04f01a92039e6cfa17aa0483ab87dcc65e0e69972c08ea8783 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 5adb3f311fae3e04f01a92039e6cfa17aa0483ab87dcc65e0e69972c08ea8783 network default NAD default NAD key ] I0514 23:16:36.406833 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 5adb3f311fae3e0, network: default, NAD default, SandboxID: "5adb3f311fae3e04f01a92039e6cfa17aa0483ab87dcc65e0e69972c08ea8783", PCI device ID: , UID: "7b9c1fee-8e15-4419-9243-8920b6c7138b", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0514 23:16:36.457072 3796 cni.go:473] [openshift-monitoring/prometheus-k8s-1 5adb3f311fae3e04f01a92039e6cfa17aa0483ab87dcc65e0e69972c08ea8783 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 5adb3f311fae3e04f01a92039e6cfa17aa0483ab87dcc65e0e69972c08ea8783 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:f7:2c:98:0a:e9\",\"name\":\"5adb3f311fae3e0\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8dc73147-f491-4e25-a751-fa23a9424b3d\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:16:38.108866 3796 base_network_controller_pods.go:487] [default/openshift-console/console-6dfb89476d-ktf8r] creating logical port openshift-console_console-6dfb89476d-ktf8r for pod on switch ip-10-0-2-152.ec2.internal I0514 23:16:38.108993 3796 kube.go:256] Updating pod openshift-console/console-6dfb89476d-ktf8r I0514 23:16:38.132584 3796 pod.go:62] [openshift-console/console-6dfb89476d-ktf8r] pod update took 23.624125ms I0514 23:16:38.132628 3796 base_network_controller_pods.go:951] [default/openshift-console/console-6dfb89476d-ktf8r] addLogicalPort annotation time took 23.676157ms I0514 23:16:38.133968 3796 pods.go:271] [openshift-console/console-6dfb89476d-ktf8r] addLogicalPort took 25.123383ms, libovsdb time 1.004127ms I0514 23:16:38.467636 3796 cni.go:452] [openshift-console/console-6dfb89476d-ktf8r f7c0d95d1d74116e2b6db1e2272b60846d15e484a72a278d47ffee5ae8481366 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6dfb89476d-ktf8r f7c0d95d1d74116e2b6db1e2272b60846d15e484a72a278d47ffee5ae8481366 network default NAD default NAD key ] I0514 23:16:38.476946 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6dfb89476d-ktf8r, hostIfaceName: f7c0d95d1d74116, network: default, NAD default, SandboxID: "f7c0d95d1d74116e2b6db1e2272b60846d15e484a72a278d47ffee5ae8481366", PCI device ID: , UID: "3b9fd85b-9e16-4a88-bafd-3eed693ee62c", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0514 23:16:38.527462 3796 cni.go:473] [openshift-console/console-6dfb89476d-ktf8r f7c0d95d1d74116e2b6db1e2272b60846d15e484a72a278d47ffee5ae8481366 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6dfb89476d-ktf8r f7c0d95d1d74116e2b6db1e2272b60846d15e484a72a278d47ffee5ae8481366 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:0b:67:07:e2:fc\",\"name\":\"f7c0d95d1d74116\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7f7b6a7f-0ef1-470c-8664-ce99191e1ae1\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:16:49.068388 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0514 23:16:52.423797 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:16:52.423820 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0514 23:16:52.423825 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0514 23:16:53.837796 3796 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:53.838414 3796 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:16:54.786835 3796 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:54.786857 3796 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:16:54.794116 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0514 23:16:54.794145 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0514 23:17:03.423446 3796 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-hvfs6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:03.423963 3796 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-hvfs6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:03.830387 3796 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-hvfs6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:03.830411 3796 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-hvfs6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:03.837440 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bf857b667-hvfs6 I0514 23:17:03.837472 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bf857b667-hvfs6 I0514 23:17:14.413894 3796 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-s2qrk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:14.414418 3796 obj_retry.go:498] Detected object openshift-console/console-7bf857b667-s2qrk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:15.020230 3796 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-s2qrk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:15.020257 3796 obj_retry.go:492] Detected object openshift-console/console-7bf857b667-s2qrk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:15.023418 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7bf857b667-s2qrk I0514 23:17:15.023446 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7bf857b667-s2qrk I0514 23:17:29.205372 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:29.205962 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:30.101794 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:30.101824 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646670-mt7xp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:17:37.297532 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0514 23:17:37.297558 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0514 23:17:38.598136 3796 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:38.598745 3796 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:17:38.970725 3796 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:38.970763 3796 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:17:38.979536 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0514 23:17:38.979561 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0514 23:19:16.141196 3796 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-28rdd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:16.141771 3796 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-28rdd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:16.332799 3796 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-28rdd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:16.332827 3796 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-28rdd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:16.341285 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6dfb89476d-28rdd I0514 23:19:16.341315 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6dfb89476d-28rdd I0514 23:19:26.672124 3796 cni.go:452] [openshift-console/console-6dfb89476d-ktf8r f7c0d95d1d74116e2b6db1e2272b60846d15e484a72a278d47ffee5ae8481366 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-6dfb89476d-ktf8r f7c0d95d1d74116e2b6db1e2272b60846d15e484a72a278d47ffee5ae8481366 network default NAD default NAD key ] I0514 23:19:26.758198 3796 cni.go:473] [openshift-console/console-6dfb89476d-ktf8r f7c0d95d1d74116e2b6db1e2272b60846d15e484a72a278d47ffee5ae8481366 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-6dfb89476d-ktf8r f7c0d95d1d74116e2b6db1e2272b60846d15e484a72a278d47ffee5ae8481366 network default NAD default NAD key default], result "{}", err I0514 23:19:26.806517 3796 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-ktf8r of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:26.806562 3796 pods.go:176] Deleting pod: openshift-console/console-6dfb89476d-ktf8r I0514 23:19:26.807838 3796 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6dfb89476d-ktf8r, ips: 10.129.0.24 I0514 23:19:26.807866 3796 obj_retry.go:498] Detected object openshift-console/console-6dfb89476d-ktf8r of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:19:27.712861 3796 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-ktf8r of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:27.712890 3796 obj_retry.go:492] Detected object openshift-console/console-6dfb89476d-ktf8r of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:19:27.723178 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6dfb89476d-ktf8r I0514 23:19:27.723208 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6dfb89476d-ktf8r I0514 23:19:38.343284 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=133 I0514 23:19:39.461460 3796 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0514 23:20:03.846211 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:20:03.846863 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:20:04.742640 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:20:04.742666 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646680-5fcxh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:20:09.427935 3796 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0514 23:20:22.342916 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=61 I0514 23:20:46.427966 3796 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0514 23:20:53.205027 3796 namespace.go:98] [tsf] adding namespace I0514 23:20:53.205887 3796 namespace.go:102] [tsf] adding namespace took 838.942µs I0514 23:20:53.221862 3796 namespace.go:142] [tsf] updating namespace I0514 23:20:53.236735 3796 namespace.go:142] [tsf] updating namespace W0514 23:20:53.952725 3796 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0514 23:20:57.613665 3796 namespace.go:98] [tsf-keycloak] adding namespace I0514 23:20:57.613896 3796 namespace.go:98] [konflux-operator] adding namespace I0514 23:20:57.613937 3796 namespace.go:98] [rhtpa-operator] adding namespace I0514 23:20:57.614121 3796 namespace.go:98] [tsf-tpa] adding namespace I0514 23:20:57.614795 3796 namespace.go:102] [tsf-keycloak] adding namespace took 1.10633ms I0514 23:20:57.615386 3796 namespace.go:102] [konflux-operator] adding namespace took 1.472451ms I0514 23:20:57.615864 3796 namespace.go:102] [rhtpa-operator] adding namespace took 1.909419ms I0514 23:20:57.616369 3796 namespace.go:102] [tsf-tpa] adding namespace took 2.230091ms I0514 23:20:57.627474 3796 namespace.go:98] [konflux-ui] adding namespace I0514 23:20:57.627556 3796 namespace.go:98] [cert-manager-operator] adding namespace I0514 23:20:57.627752 3796 namespace.go:98] [tsf-tas] adding namespace I0514 23:20:57.628230 3796 namespace.go:102] [konflux-ui] adding namespace took 742.517µs I0514 23:20:57.628811 3796 namespace.go:102] [cert-manager-operator] adding namespace took 1.240361ms I0514 23:20:57.628839 3796 namespace.go:98] [rhbk-operator] adding namespace I0514 23:20:57.629412 3796 namespace.go:102] [tsf-tas] adding namespace took 1.64732ms I0514 23:20:57.629925 3796 namespace.go:102] [rhbk-operator] adding namespace took 1.079125ms I0514 23:20:57.633198 3796 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:57.645145 3796 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:57.679902 3796 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:57.680000 3796 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:20:57.680122 3796 namespace.go:142] [konflux-operator] updating namespace I0514 23:20:57.680204 3796 namespace.go:142] [konflux-ui] updating namespace I0514 23:20:57.694894 3796 namespace.go:142] [konflux-operator] updating namespace I0514 23:20:57.694935 3796 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:20:57.695069 3796 namespace.go:142] [rhbk-operator] updating namespace I0514 23:20:57.695150 3796 namespace.go:142] [tsf-tas] updating namespace I0514 23:20:57.695209 3796 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:20:57.720899 3796 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:57.736768 3796 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:57.743748 3796 namespace.go:142] [konflux-ui] updating namespace I0514 23:20:57.749755 3796 namespace.go:142] [konflux-ui] updating namespace I0514 23:20:57.752779 3796 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:20:57.773868 3796 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:20:57.773899 3796 namespace.go:142] [rhbk-operator] updating namespace I0514 23:20:57.784718 3796 namespace.go:142] [rhbk-operator] updating namespace I0514 23:20:57.792672 3796 namespace.go:142] [tsf-tas] updating namespace I0514 23:20:57.807295 3796 namespace.go:142] [tsf-tas] updating namespace I0514 23:20:57.807337 3796 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:20:57.827123 3796 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:20:57.841974 3796 namespace.go:142] [konflux-operator] updating namespace I0514 23:20:58.809554 3796 namespace.go:142] [tsf-tpa] updating namespace I0514 23:20:58.818104 3796 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:21:00.467481 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0514 23:21:05.004943 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0514 23:21:05.005016 3796 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-l9rdx IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-l9rdx: no pod IPs found I0514 23:21:06.263695 3796 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.264262 3796 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.264289 3796 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.264300 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:06.264315 3796 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:06.264321 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:07.292897 3796 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.292942 3796 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.292955 3796 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:07.292965 3796 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51xxtpd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:11.205632 3796 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.206183 3796 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.206211 3796 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.206219 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:11.206231 3796 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:11.206237 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:12.233735 3796 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:12.233764 3796 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:12.233777 3796 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:12.233786 3796 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7f9db of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:14.237139 3796 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.237678 3796 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.237710 3796 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.237721 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:14.237734 3796 obj_retry.go:498] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:14.237739 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.253650 3796 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.254195 3796 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.254229 3796 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.254238 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:15.254247 3796 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.254251 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.265813 3796 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.265834 3796 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.265849 3796 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.265858 3796 obj_retry.go:492] Detected object openshift-marketplace/5bfc73bccd2cef20bb9afe324d112069530a83adee9f41e094fba66f6ew2qlv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:15.316246 3796 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.316835 3796 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.316860 3796 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.316868 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:15.316879 3796 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.316883 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.340298 3796 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.340752 3796 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.340772 3796 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.340778 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0514 23:21:15.340791 3796 obj_retry.go:498] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0514 23:21:15.340795 3796 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0514 23:21:16.284620 3796 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.284647 3796 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.284667 3796 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.284684 3796 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7dd4ww of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.341946 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0514 23:21:16.343450 3796 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.343475 3796 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.343490 3796 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.343511 3796 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c213655gtg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.371110 3796 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.371135 3796 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.371153 3796 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:21:16.371165 3796 obj_retry.go:492] Detected object openshift-marketplace/5aa5e4cbc2f6ac9107b7dd5bfb7c8725ca307e639be07100d6806658fe4cmgw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:21:16.801699 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.501835 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.531243 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.540806 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0514 23:21:20.577421 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0514 23:21:29.149963 3796 namespace.go:98] [cert-manager] adding namespace I0514 23:21:29.150985 3796 namespace.go:102] [cert-manager] adding namespace took 995.813µs I0514 23:21:29.166833 3796 namespace.go:142] [cert-manager] updating namespace I0514 23:21:29.216195 3796 namespace.go:142] [cert-manager] updating namespace I0514 23:21:29.727877 3796 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-wbhn6] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-wbhn6 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:21:29.727971 3796 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-wbhn6 I0514 23:21:29.752803 3796 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-wbhn6] pod update took 24.857891ms I0514 23:21:29.752824 3796 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-wbhn6] addLogicalPort annotation time took 24.882707ms I0514 23:21:29.753886 3796 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-wbhn6] addLogicalPort took 26.02568ms, libovsdb time 761.281µs I0514 23:21:30.099349 3796 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-wbhn6 63025f3e6ba68323d2e85a5dffe439e54a99b41229b9ef1c7e668de1baffa408 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-wbhn6 63025f3e6ba68323d2e85a5dffe439e54a99b41229b9ef1c7e668de1baffa408 network default NAD default NAD key ] I0514 23:21:30.109078 3796 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-wbhn6, hostIfaceName: 63025f3e6ba6832, network: default, NAD default, SandboxID: "63025f3e6ba68323d2e85a5dffe439e54a99b41229b9ef1c7e668de1baffa408", PCI device ID: , UID: "8ff0bf8b-3aa3-46e1-a2af-3bd310b6493c", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0514 23:21:30.158788 3796 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-wbhn6 63025f3e6ba68323d2e85a5dffe439e54a99b41229b9ef1c7e668de1baffa408 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-wbhn6 63025f3e6ba68323d2e85a5dffe439e54a99b41229b9ef1c7e668de1baffa408 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:16:06:5d:37:4a\",\"name\":\"63025f3e6ba6832\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f96e371-50a0-40f5-b75f-95f3bf05d871\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:21:33.243424 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0514 23:21:33.243440 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0514 23:21:38.684182 3796 namespace.go:142] [openshift-operators] updating namespace I0514 23:21:38.703180 3796 namespace.go:142] [openshift-operators] updating namespace W0514 23:21:39.254479 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0514 23:21:39.254499 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0514 23:21:39.703749 3796 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59c77684cc-4k9vq] creating logical port cert-manager_cert-manager-59c77684cc-4k9vq for pod on switch ip-10-0-2-152.ec2.internal I0514 23:21:39.703847 3796 kube.go:256] Updating pod cert-manager/cert-manager-59c77684cc-4k9vq I0514 23:21:39.720403 3796 pod.go:62] [cert-manager/cert-manager-59c77684cc-4k9vq] pod update took 16.580002ms I0514 23:21:39.720430 3796 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-59c77684cc-4k9vq] addLogicalPort annotation time took 16.612667ms I0514 23:21:39.721775 3796 pods.go:271] [cert-manager/cert-manager-59c77684cc-4k9vq] addLogicalPort took 18.041866ms, libovsdb time 930.858µs I0514 23:21:40.066319 3796 cni.go:452] [cert-manager/cert-manager-59c77684cc-4k9vq 8a123c169604d7dffbe63d8d2254fcc0d0539f073fda8bb2ade2244b343c55b6 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-59c77684cc-4k9vq 8a123c169604d7dffbe63d8d2254fcc0d0539f073fda8bb2ade2244b343c55b6 network default NAD default NAD key ] I0514 23:21:40.075660 3796 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59c77684cc-4k9vq, hostIfaceName: 8a123c169604d7d, network: default, NAD default, SandboxID: "8a123c169604d7dffbe63d8d2254fcc0d0539f073fda8bb2ade2244b343c55b6", PCI device ID: , UID: "e93b66d5-fe83-45ae-9015-308ea66ab0de", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0514 23:21:40.126256 3796 cni.go:473] [cert-manager/cert-manager-59c77684cc-4k9vq 8a123c169604d7dffbe63d8d2254fcc0d0539f073fda8bb2ade2244b343c55b6 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-59c77684cc-4k9vq 8a123c169604d7dffbe63d8d2254fcc0d0539f073fda8bb2ade2244b343c55b6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:87:f8:70:1e:d1\",\"name\":\"8a123c169604d7d\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dc6d5158-8880-4b89-9ff4-9effb07aee54\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:21:44.340415 3796 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-mgln8] creating logical port openshift-operators_rhtas-operator-controller-manager-68888cb75-mgln8 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:21:44.340526 3796 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-68888cb75-mgln8 I0514 23:21:44.360407 3796 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-68888cb75-mgln8] pod update took 19.907428ms I0514 23:21:44.360428 3796 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-mgln8] addLogicalPort annotation time took 19.935272ms I0514 23:21:44.361680 3796 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-68888cb75-mgln8] addLogicalPort took 21.281114ms, libovsdb time 856.482µs I0514 23:21:44.713866 3796 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-68888cb75-mgln8 86d87a7b07686adda58420dcd429b42e3d832cc812eda8f8995953ecc9f6a888 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-mgln8 86d87a7b07686adda58420dcd429b42e3d832cc812eda8f8995953ecc9f6a888 network default NAD default NAD key ] I0514 23:21:44.723311 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-68888cb75-mgln8, hostIfaceName: 86d87a7b07686ad, network: default, NAD default, SandboxID: "86d87a7b07686adda58420dcd429b42e3d832cc812eda8f8995953ecc9f6a888", PCI device ID: , UID: "1486d143-8537-438b-bbbc-21d69886f688", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0514 23:21:44.774240 3796 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-68888cb75-mgln8 86d87a7b07686adda58420dcd429b42e3d832cc812eda8f8995953ecc9f6a888 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-mgln8 86d87a7b07686adda58420dcd429b42e3d832cc812eda8f8995953ecc9f6a888 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:cf:64:2a:bb:be\",\"name\":\"86d87a7b07686ad\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/29fdda0d-80d7-491e-9bee-945da61f6c59\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:21:44.964127 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0514 23:21:47.311806 3796 namespace.go:98] [trusted-artifact-signer] adding namespace I0514 23:21:47.312657 3796 namespace.go:102] [trusted-artifact-signer] adding namespace took 828.161µs W0514 23:21:47.338045 3796 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0514 23:21:47.378902 3796 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:21:47.395911 3796 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:21:48.498032 3796 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-7d56d9fd65-cccb6] creating logical port trusted-artifact-signer_cli-server-7d56d9fd65-cccb6 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:21:48.498122 3796 kube.go:256] Updating pod trusted-artifact-signer/cli-server-7d56d9fd65-cccb6 I0514 23:21:48.554900 3796 pod.go:62] [trusted-artifact-signer/cli-server-7d56d9fd65-cccb6] pod update took 56.799344ms I0514 23:21:48.554925 3796 base_network_controller_pods.go:951] [default/trusted-artifact-signer/cli-server-7d56d9fd65-cccb6] addLogicalPort annotation time took 56.827967ms I0514 23:21:48.556097 3796 pods.go:271] [trusted-artifact-signer/cli-server-7d56d9fd65-cccb6] addLogicalPort took 58.088921ms, libovsdb time 836.838µs I0514 23:21:48.846670 3796 cni.go:452] [trusted-artifact-signer/cli-server-7d56d9fd65-cccb6 be02986f335cca1f9f6fcfcdbbe8aa22453583cc4c59297dfb15c5023423960c network default NAD default NAD key ] ADD starting CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-cccb6 be02986f335cca1f9f6fcfcdbbe8aa22453583cc4c59297dfb15c5023423960c network default NAD default NAD key ] I0514 23:21:48.856032 3796 helper_linux.go:556] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-7d56d9fd65-cccb6, hostIfaceName: be02986f335cca1, network: default, NAD default, SandboxID: "be02986f335cca1f9f6fcfcdbbe8aa22453583cc4c59297dfb15c5023423960c", PCI device ID: , UID: "f93284d1-abc7-485d-bd43-2acaedaea1db", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0514 23:21:48.907879 3796 cni.go:473] [trusted-artifact-signer/cli-server-7d56d9fd65-cccb6 be02986f335cca1f9f6fcfcdbbe8aa22453583cc4c59297dfb15c5023423960c network default NAD default NAD key default] ADD finished CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-cccb6 be02986f335cca1f9f6fcfcdbbe8aa22453583cc4c59297dfb15c5023423960c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fe:a1:02:30:75:a1\",\"name\":\"be02986f335cca1\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/37870680-b032-4b12-8320-2d8e5ec08b77\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:21:51.477987 3796 namespace.go:142] [openshift-operators] updating namespace I0514 23:21:52.368377 3796 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-jksvc] creating logical port openshift-operators_openshift-pipelines-operator-6bdf6c8794-jksvc for pod on switch ip-10-0-2-152.ec2.internal I0514 23:21:52.368537 3796 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-jksvc I0514 23:21:52.491670 3796 pod.go:62] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-jksvc] pod update took 123.173972ms I0514 23:21:52.491721 3796 base_network_controller_pods.go:951] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-jksvc] addLogicalPort annotation time took 123.228691ms I0514 23:21:52.493199 3796 pods.go:271] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-jksvc] addLogicalPort took 124.84695ms, libovsdb time 1.030925ms I0514 23:21:54.148774 3796 cni.go:452] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-jksvc 03175718bcb1a27f649e86ececf0781dcad7822aa7ee8a8ea16fea97b6632c1c network default NAD default NAD key ] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-jksvc 03175718bcb1a27f649e86ececf0781dcad7822aa7ee8a8ea16fea97b6632c1c network default NAD default NAD key ] I0514 23:21:54.158338 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6bdf6c8794-jksvc, hostIfaceName: 03175718bcb1a27, network: default, NAD default, SandboxID: "03175718bcb1a27f649e86ececf0781dcad7822aa7ee8a8ea16fea97b6632c1c", PCI device ID: , UID: "62ace383-490f-4a62-97d0-ba0bdf4646e8", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0514 23:21:54.213201 3796 cni.go:473] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-jksvc 03175718bcb1a27f649e86ececf0781dcad7822aa7ee8a8ea16fea97b6632c1c network default NAD default NAD key default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-jksvc 03175718bcb1a27f649e86ececf0781dcad7822aa7ee8a8ea16fea97b6632c1c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:6a:5b:4b:e4:75\",\"name\":\"03175718bcb1a27\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/89fdd4fc-e169-435e-8b7c-a38295159954\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:21:58.439982 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0514 23:22:01.467348 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0514 23:22:03.850016 3796 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:03.850631 3796 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:03.880300 3796 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:03.880387 3796 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:03.919326 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0514 23:22:03.919359 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0514 23:22:08.866681 3796 namespace.go:98] [openshift-pipelines] adding namespace I0514 23:22:08.867455 3796 namespace.go:102] [openshift-pipelines] adding namespace took 751.324µs I0514 23:22:08.883873 3796 namespace.go:142] [openshift-pipelines] updating namespace I0514 23:22:09.703504 3796 namespace.go:142] [cert-manager] updating namespace I0514 23:22:09.901976 3796 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:22:10.305022 3796 namespace.go:142] [konflux-operator] updating namespace I0514 23:22:10.432476 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0514 23:22:10.455529 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0514 23:22:10.504139 3796 namespace.go:142] [konflux-ui] updating namespace I0514 23:22:10.704917 3796 namespace.go:142] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] updating namespace I0514 23:22:11.109257 3796 namespace.go:142] [rhbk-operator] updating namespace I0514 23:22:11.307920 3796 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:22:11.510411 3796 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:22:11.711010 3796 namespace.go:142] [tsf] updating namespace I0514 23:22:11.912285 3796 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:22:12.113272 3796 namespace.go:142] [tsf-tas] updating namespace I0514 23:22:12.313009 3796 namespace.go:142] [tsf-tpa] updating namespace I0514 23:22:12.451195 3796 namespace.go:142] [cert-manager] updating namespace I0514 23:22:12.461176 3796 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:22:12.471104 3796 namespace.go:142] [default] updating namespace I0514 23:22:12.481350 3796 namespace.go:142] [konflux-operator] updating namespace I0514 23:22:12.492434 3796 namespace.go:142] [konflux-ui] updating namespace I0514 23:22:12.503844 3796 namespace.go:142] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] updating namespace I0514 23:22:12.512830 3796 namespace.go:142] [openshift] updating namespace I0514 23:22:12.523720 3796 namespace.go:142] [rhbk-operator] updating namespace I0514 23:22:12.534822 3796 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:22:12.544481 3796 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:22:12.554995 3796 namespace.go:142] [tsf] updating namespace I0514 23:22:12.565834 3796 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:22:12.577986 3796 namespace.go:142] [tsf-tas] updating namespace I0514 23:22:12.597339 3796 namespace.go:142] [tsf-tpa] updating namespace I0514 23:22:12.698544 3796 namespace.go:142] [cert-manager] updating namespace I0514 23:22:12.800856 3796 namespace.go:142] [cert-manager-operator] updating namespace I0514 23:22:12.899962 3796 namespace.go:142] [default] updating namespace I0514 23:22:13.003341 3796 namespace.go:142] [konflux-operator] updating namespace I0514 23:22:13.098974 3796 namespace.go:142] [konflux-ui] updating namespace I0514 23:22:13.201213 3796 namespace.go:142] [open-cluster-management-2qa0bb9570tfkc4jihthla6gq803g80l] updating namespace I0514 23:22:13.296663 3796 namespace.go:142] [openshift] updating namespace I0514 23:22:13.398562 3796 namespace.go:142] [rhbk-operator] updating namespace I0514 23:22:13.497700 3796 namespace.go:142] [rhtpa-operator] updating namespace I0514 23:22:13.597439 3796 namespace.go:142] [trusted-artifact-signer] updating namespace I0514 23:22:13.695871 3796 namespace.go:142] [tsf] updating namespace I0514 23:22:13.796557 3796 namespace.go:142] [tsf-keycloak] updating namespace I0514 23:22:13.896072 3796 namespace.go:142] [tsf-tas] updating namespace I0514 23:22:13.999146 3796 namespace.go:142] [tsf-tpa] updating namespace I0514 23:22:14.255974 3796 namespace.go:142] [openshift-pipelines] updating namespace W0514 23:22:18.461232 3796 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0514 23:22:18.488977 3796 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0514 23:22:21.871580 3796 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g for pod on switch ip-10-0-2-152.ec2.internal I0514 23:22:21.871745 3796 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g I0514 23:22:21.891199 3796 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g] pod update took 19.526252ms I0514 23:22:21.891227 3796 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g] addLogicalPort annotation time took 19.561918ms I0514 23:22:21.892699 3796 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g] addLogicalPort took 21.141901ms, libovsdb time 861.033µs I0514 23:22:21.962234 3796 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-xbg2k] creating logical port openshift-pipelines_tekton-pipelines-webhook-7b6d6f4c94-xbg2k for pod on switch ip-10-0-2-152.ec2.internal I0514 23:22:21.962332 3796 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-xbg2k I0514 23:22:21.979809 3796 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-xbg2k] pod update took 17.504638ms I0514 23:22:21.979835 3796 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-xbg2k] addLogicalPort annotation time took 17.53577ms I0514 23:22:21.980968 3796 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-xbg2k] addLogicalPort took 18.754302ms, libovsdb time 775.722µs I0514 23:22:22.242930 3796 cni.go:452] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g 5e19e91ee8c3243ed90062a1da2ad3e11f93b3078d12df310d560497de93c4b3 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g 5e19e91ee8c3243ed90062a1da2ad3e11f93b3078d12df310d560497de93c4b3 network default NAD default NAD key ] I0514 23:22:22.253383 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g, hostIfaceName: 5e19e91ee8c3243, network: default, NAD default, SandboxID: "5e19e91ee8c3243ed90062a1da2ad3e11f93b3078d12df310d560497de93c4b3", PCI device ID: , UID: "222df01e-da49-445d-a9ec-ffbd16a27468", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0514 23:22:22.309936 3796 cni.go:473] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g 5e19e91ee8c3243ed90062a1da2ad3e11f93b3078d12df310d560497de93c4b3 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-gtx7g 5e19e91ee8c3243ed90062a1da2ad3e11f93b3078d12df310d560497de93c4b3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:a3:7f:26:ce:26\",\"name\":\"5e19e91ee8c3243\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f7d6fbfd-8048-4733-b868-b8a317a76489\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:22:22.326990 3796 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-xbg2k 62c4afd188949841c920f6886d41e3885e93ba6d1bee383fd480f6ccb05ca718 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-xbg2k 62c4afd188949841c920f6886d41e3885e93ba6d1bee383fd480f6ccb05ca718 network default NAD default NAD key ] I0514 23:22:22.338147 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-7b6d6f4c94-xbg2k, hostIfaceName: 62c4afd18894984, network: default, NAD default, SandboxID: "62c4afd188949841c920f6886d41e3885e93ba6d1bee383fd480f6ccb05ca718", PCI device ID: , UID: "45a55bd5-af40-4f68-adc4-518ba215cc0f", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0514 23:22:22.393351 3796 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-xbg2k 62c4afd188949841c920f6886d41e3885e93ba6d1bee383fd480f6ccb05ca718 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-xbg2k 62c4afd188949841c920f6886d41e3885e93ba6d1bee383fd480f6ccb05ca718 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:1b:98:05:36:ea\",\"name\":\"62c4afd18894984\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0ca5ab30-248a-4eb5-96e3-ce0a231a1715\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:22:26.153997 3796 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:26.154019 3796 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:26.154027 3796 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0514 23:22:26.191519 3796 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:26.191543 3796 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:26.191550 3796 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0514 23:22:26.536163 3796 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:26.536190 3796 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:26.536197 3796 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:26.536203 3796 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:37.527491 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:37.527513 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:37.527518 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0514 23:22:37.527524 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0514 23:22:42.732064 3796 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:42.732677 3796 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:22:42.752469 3796 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:42.752489 3796 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:22:42.759171 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0514 23:22:42.759195 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0514 23:22:44.807377 3796 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0514 23:22:44.808191 3796 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0514 23:22:44.808212 3796 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0514 23:22:44.810385 3796 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0514 23:22:44.878874 3796 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-zcr4s] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-zcr4s for pod on switch ip-10-0-2-152.ec2.internal I0514 23:22:44.879023 3796 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-zcr4s I0514 23:22:44.897454 3796 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-zcr4s] pod update took 18.456863ms I0514 23:22:44.897474 3796 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-zcr4s] addLogicalPort annotation time took 18.483179ms I0514 23:22:44.898847 3796 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-zcr4s] addLogicalPort took 19.987561ms, libovsdb time 966.369µs I0514 23:22:45.239616 3796 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-zcr4s f2ecea24daa76d4894eefa85f9ee1b67eac95acb5dbf714cc4d2b3dea8123bbe network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-zcr4s f2ecea24daa76d4894eefa85f9ee1b67eac95acb5dbf714cc4d2b3dea8123bbe network default NAD default NAD key ] I0514 23:22:45.249279 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-zcr4s, hostIfaceName: f2ecea24daa76d4, network: default, NAD default, SandboxID: "f2ecea24daa76d4894eefa85f9ee1b67eac95acb5dbf714cc4d2b3dea8123bbe", PCI device ID: , UID: "8bdc4875-123e-45d5-ad9a-83b7386e6c33", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0514 23:22:45.302119 3796 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-zcr4s f2ecea24daa76d4894eefa85f9ee1b67eac95acb5dbf714cc4d2b3dea8123bbe network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-zcr4s f2ecea24daa76d4894eefa85f9ee1b67eac95acb5dbf714cc4d2b3dea8123bbe network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:b8:99:e9:c5:d2\",\"name\":\"f2ecea24daa76d4\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/040920a8-3d90-405b-9df4-c55ee6575811\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:22:46.342847 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=107 W0514 23:22:47.626851 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0514 23:22:53.344864 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=105 W0514 23:22:55.702181 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0514 23:22:55.702202 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0514 23:22:55.702255 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0514 23:22:55.702273 3796 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0514 23:22:58.632756 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0514 23:23:05.320707 3796 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0514 23:23:08.403546 3796 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:23:08.403657 3796 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0514 23:23:08.417656 3796 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 14.020257ms I0514 23:23:08.417686 3796 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 14.055108ms I0514 23:23:08.419147 3796 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 15.615438ms, libovsdb time 938.328µs I0514 23:23:13.272312 3796 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 bb1e85df2886f3414f6d065d3053cdbac457551795b85776c18b276a60e743ad network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 bb1e85df2886f3414f6d065d3053cdbac457551795b85776c18b276a60e743ad network default NAD default NAD key ] I0514 23:23:13.282678 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: bb1e85df2886f34, network: default, NAD default, SandboxID: "bb1e85df2886f3414f6d065d3053cdbac457551795b85776c18b276a60e743ad", PCI device ID: , UID: "8adbf476-a48d-4f92-8b15-314a99bec52a", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0514 23:23:13.337276 3796 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 bb1e85df2886f3414f6d065d3053cdbac457551795b85776c18b276a60e743ad network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 bb1e85df2886f3414f6d065d3053cdbac457551795b85776c18b276a60e743ad network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:e1:34:73:c6:db\",\"name\":\"bb1e85df2886f34\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e73f13ec-ff0a-4311-bfaf-f55f229d08c7\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:23:13.450735 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0514 23:23:18.819665 3796 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0514 23:23:20.184166 3796 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-64c5c8474c-rrhlk] creating logical port openshift-pipelines_tekton-results-api-64c5c8474c-rrhlk for pod on switch ip-10-0-2-152.ec2.internal I0514 23:23:20.184287 3796 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-64c5c8474c-rrhlk I0514 23:23:20.201369 3796 pod.go:62] [openshift-pipelines/tekton-results-api-64c5c8474c-rrhlk] pod update took 17.10596ms I0514 23:23:20.201391 3796 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-api-64c5c8474c-rrhlk] addLogicalPort annotation time took 17.134162ms I0514 23:23:20.202687 3796 pods.go:271] [openshift-pipelines/tekton-results-api-64c5c8474c-rrhlk] addLogicalPort took 18.539441ms, libovsdb time 911.253µs I0514 23:23:20.276288 3796 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-zv2xc] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-585694b54-zv2xc for pod on switch ip-10-0-2-152.ec2.internal I0514 23:23:20.276389 3796 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-585694b54-zv2xc I0514 23:23:20.291570 3796 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-zv2xc] pod update took 15.205448ms I0514 23:23:20.291591 3796 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-retention-policy-agent-585694b54-zv2xc] addLogicalPort annotation time took 15.231844ms I0514 23:23:20.292852 3796 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-zv2xc] addLogicalPort took 16.583421ms, libovsdb time 869.534µs I0514 23:23:20.546554 3796 cni.go:452] [openshift-pipelines/tekton-results-api-64c5c8474c-rrhlk ef83a62b83bad78765bfdddad221f388177766b0844a337b982dce421a3cbef5 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-rrhlk ef83a62b83bad78765bfdddad221f388177766b0844a337b982dce421a3cbef5 network default NAD default NAD key ] I0514 23:23:20.556048 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-64c5c8474c-rrhlk, hostIfaceName: ef83a62b83bad78, network: default, NAD default, SandboxID: "ef83a62b83bad78765bfdddad221f388177766b0844a337b982dce421a3cbef5", PCI device ID: , UID: "1ed1e729-7f24-450b-96ba-1ee00708bd91", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0514 23:23:20.610649 3796 cni.go:473] [openshift-pipelines/tekton-results-api-64c5c8474c-rrhlk ef83a62b83bad78765bfdddad221f388177766b0844a337b982dce421a3cbef5 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-rrhlk ef83a62b83bad78765bfdddad221f388177766b0844a337b982dce421a3cbef5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"26:e6:d0:cb:32:15\",\"name\":\"ef83a62b83bad78\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/333b80e1-6487-4a33-93b2-c0a21282d373\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:23:20.636563 3796 cni.go:452] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-zv2xc b2303f48bf3538eece03793e96c9d7e1f570ddc4fe248df5b8269ad23fe691da network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-zv2xc b2303f48bf3538eece03793e96c9d7e1f570ddc4fe248df5b8269ad23fe691da network default NAD default NAD key ] I0514 23:23:20.646498 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-585694b54-zv2xc, hostIfaceName: b2303f48bf3538e, network: default, NAD default, SandboxID: "b2303f48bf3538eece03793e96c9d7e1f570ddc4fe248df5b8269ad23fe691da", PCI device ID: , UID: "7de12a48-e8e5-4648-ad77-9ea6b7084fff", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0514 23:23:20.698505 3796 cni.go:473] [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-zv2xc b2303f48bf3538eece03793e96c9d7e1f570ddc4fe248df5b8269ad23fe691da network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-585694b54-zv2xc b2303f48bf3538eece03793e96c9d7e1f570ddc4fe248df5b8269ad23fe691da network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:c9:6f:c1:55:09\",\"name\":\"b2303f48bf3538e\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/278d9ff4-7e22-47ed-bad5-d11e0f881936\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:23:25.839584 3796 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0514 23:23:25.839623 3796 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0514 23:23:25.839632 3796 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0514 23:23:28.357505 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=851 W0514 23:23:30.840327 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0514 23:23:30.840353 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0514 23:23:30.840361 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0514 23:23:35.096429 3796 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:35.096979 3796 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:35.141355 3796 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:35.141380 3796 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:35.148588 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0514 23:23:35.148641 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0514 23:23:37.577040 3796 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-844d67b8b5-49l72] creating logical port tsf-tas_fulcio-server-844d67b8b5-49l72 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:23:37.577137 3796 kube.go:256] Updating pod tsf-tas/fulcio-server-844d67b8b5-49l72 I0514 23:23:37.596625 3796 pod.go:62] [tsf-tas/fulcio-server-844d67b8b5-49l72] pod update took 19.499812ms I0514 23:23:37.596682 3796 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-844d67b8b5-49l72] addLogicalPort annotation time took 19.57812ms I0514 23:23:37.597960 3796 pods.go:271] [tsf-tas/fulcio-server-844d67b8b5-49l72] addLogicalPort took 20.941595ms, libovsdb time 876.137µs I0514 23:23:37.938254 3796 cni.go:452] [tsf-tas/fulcio-server-844d67b8b5-49l72 d64c0e4f3ba0a0407979081409208e17c47c78d827eaa973741fcc6ac9f8bf0a network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-844d67b8b5-49l72 d64c0e4f3ba0a0407979081409208e17c47c78d827eaa973741fcc6ac9f8bf0a network default NAD default NAD key ] I0514 23:23:37.947947 3796 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-844d67b8b5-49l72, hostIfaceName: d64c0e4f3ba0a04, network: default, NAD default, SandboxID: "d64c0e4f3ba0a0407979081409208e17c47c78d827eaa973741fcc6ac9f8bf0a", PCI device ID: , UID: "9d684508-1e9d-4d75-b6a6-a2401329f966", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0514 23:23:38.006656 3796 cni.go:473] [tsf-tas/fulcio-server-844d67b8b5-49l72 d64c0e4f3ba0a0407979081409208e17c47c78d827eaa973741fcc6ac9f8bf0a network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-844d67b8b5-49l72 d64c0e4f3ba0a0407979081409208e17c47c78d827eaa973741fcc6ac9f8bf0a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:78:cf:76:21:31\",\"name\":\"d64c0e4f3ba0a04\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5a043413-c3e9-4380-8cc4-5481e3a717c3\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:23:38.108348 3796 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-7p2l9-j2vbg] creating logical port tsf-tas_rekor-createtree-job-7p2l9-j2vbg for pod on switch ip-10-0-2-152.ec2.internal I0514 23:23:38.108461 3796 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-7p2l9-j2vbg I0514 23:23:38.126412 3796 pod.go:62] [tsf-tas/rekor-createtree-job-7p2l9-j2vbg] pod update took 17.982231ms I0514 23:23:38.126437 3796 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-7p2l9-j2vbg] addLogicalPort annotation time took 18.01299ms I0514 23:23:38.127739 3796 pods.go:271] [tsf-tas/rekor-createtree-job-7p2l9-j2vbg] addLogicalPort took 19.413183ms, libovsdb time 817.452µs I0514 23:23:38.474072 3796 cni.go:452] [tsf-tas/rekor-createtree-job-7p2l9-j2vbg 184ebfb2904fdf8eec09b5e8edc7302d037172b8b064eabfd157601c950ba4d4 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-7p2l9-j2vbg 184ebfb2904fdf8eec09b5e8edc7302d037172b8b064eabfd157601c950ba4d4 network default NAD default NAD key ] I0514 23:23:38.483795 3796 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-7p2l9-j2vbg, hostIfaceName: 184ebfb2904fdf8, network: default, NAD default, SandboxID: "184ebfb2904fdf8eec09b5e8edc7302d037172b8b064eabfd157601c950ba4d4", PCI device ID: , UID: "f1480736-4309-46a7-983d-ca215454964d", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0514 23:23:38.539224 3796 cni.go:473] [tsf-tas/rekor-createtree-job-7p2l9-j2vbg 184ebfb2904fdf8eec09b5e8edc7302d037172b8b064eabfd157601c950ba4d4 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-7p2l9-j2vbg 184ebfb2904fdf8eec09b5e8edc7302d037172b8b064eabfd157601c950ba4d4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:77:97:57:a8:8d\",\"name\":\"184ebfb2904fdf8\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/72d96fc0-39f5-453e-91b3-ab80abff12c3\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:23:40.795597 3796 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-5c4496c4cc-bgzhg] creating logical port tsf-tas_trillian-db-5c4496c4cc-bgzhg for pod on switch ip-10-0-2-152.ec2.internal I0514 23:23:40.795703 3796 kube.go:256] Updating pod tsf-tas/trillian-db-5c4496c4cc-bgzhg I0514 23:23:40.808968 3796 pod.go:62] [tsf-tas/trillian-db-5c4496c4cc-bgzhg] pod update took 13.286747ms I0514 23:23:40.809000 3796 base_network_controller_pods.go:951] [default/tsf-tas/trillian-db-5c4496c4cc-bgzhg] addLogicalPort annotation time took 13.323794ms I0514 23:23:40.810393 3796 pods.go:271] [tsf-tas/trillian-db-5c4496c4cc-bgzhg] addLogicalPort took 14.813723ms, libovsdb time 915.504µs W0514 23:23:40.890945 3796 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0514 23:23:40.890967 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0514 23:23:40.916711 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0514 23:23:40.916731 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server W0514 23:23:46.036528 3796 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0514 23:23:46.163846 3796 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-6ccvn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:46.164403 3796 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-6ccvn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:47.068152 3796 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-6ccvn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:47.068175 3796 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-6ccvn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:50.176031 3796 cni.go:452] [tsf-tas/trillian-db-5c4496c4cc-bgzhg 37f7eb7bcf4cc013535c36c7b4b194fbff9fe74a7e164b7a14bce25b7bea0dd6 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-db-5c4496c4cc-bgzhg 37f7eb7bcf4cc013535c36c7b4b194fbff9fe74a7e164b7a14bce25b7bea0dd6 network default NAD default NAD key ] I0514 23:23:50.185546 3796 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-5c4496c4cc-bgzhg, hostIfaceName: 37f7eb7bcf4cc01, network: default, NAD default, SandboxID: "37f7eb7bcf4cc013535c36c7b4b194fbff9fe74a7e164b7a14bce25b7bea0dd6", PCI device ID: , UID: "6cca075c-20ea-4ae7-8f2c-d495485b23bd", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0514 23:23:50.240817 3796 cni.go:473] [tsf-tas/trillian-db-5c4496c4cc-bgzhg 37f7eb7bcf4cc013535c36c7b4b194fbff9fe74a7e164b7a14bce25b7bea0dd6 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-db-5c4496c4cc-bgzhg 37f7eb7bcf4cc013535c36c7b4b194fbff9fe74a7e164b7a14bce25b7bea0dd6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:6c:dc:ca:cf:f9\",\"name\":\"37f7eb7bcf4cc01\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7c7e2d12-a1b5-46ec-9603-6f33606d2cba\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:23:53.959924 3796 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0514 23:23:55.613762 3796 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0514 23:23:57.219923 3796 namespace.go:142] [openshift-pipelines] updating namespace I0514 23:23:58.741775 3796 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:58.742313 3796 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:23:59.760264 3796 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:23:59.760290 3796 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-4d45n-nzds2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:03.438410 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 W0514 23:24:04.155129 3796 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0514 23:24:05.004267 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql I0514 23:24:16.245916 3796 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-mnlc7] creating logical port openshift-pipelines_pipelines-console-plugin-65f788bc74-mnlc7 for pod on switch ip-10-0-2-152.ec2.internal I0514 23:24:16.246058 3796 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-65f788bc74-mnlc7 I0514 23:24:16.261552 3796 pod.go:62] [openshift-pipelines/pipelines-console-plugin-65f788bc74-mnlc7] pod update took 15.518524ms I0514 23:24:16.261573 3796 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-mnlc7] addLogicalPort annotation time took 15.544752ms I0514 23:24:16.262946 3796 pods.go:271] [openshift-pipelines/pipelines-console-plugin-65f788bc74-mnlc7] addLogicalPort took 17.046731ms, libovsdb time 899.926µs I0514 23:24:16.606473 3796 cni.go:452] [openshift-pipelines/pipelines-console-plugin-65f788bc74-mnlc7 118af7a646b4980ec5d13f2921a330cdeee6dce022084cf8cb0f2085456ed8b9 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-mnlc7 118af7a646b4980ec5d13f2921a330cdeee6dce022084cf8cb0f2085456ed8b9 network default NAD default NAD key ] I0514 23:24:16.616179 3796 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-65f788bc74-mnlc7, hostIfaceName: 118af7a646b4980, network: default, NAD default, SandboxID: "118af7a646b4980ec5d13f2921a330cdeee6dce022084cf8cb0f2085456ed8b9", PCI device ID: , UID: "5c85ee19-2bec-4ff4-9dd4-b66e01b7522e", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0514 23:24:16.670424 3796 cni.go:473] [openshift-pipelines/pipelines-console-plugin-65f788bc74-mnlc7 118af7a646b4980ec5d13f2921a330cdeee6dce022084cf8cb0f2085456ed8b9 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-mnlc7 118af7a646b4980ec5d13f2921a330cdeee6dce022084cf8cb0f2085456ed8b9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:ce:94:35:ea:18\",\"name\":\"118af7a646b4980\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7a3e8617-30f8-4130-ad3b-7fd8f5fc4d49\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:24:24.922311 3796 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:24.922925 3796 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:25.444149 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0514 23:24:25.766363 3796 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:25.766391 3796 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-tsvkq-4kksv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:27.429171 3796 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:27.429812 3796 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:27.450111 3796 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:27.450133 3796 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:27.457185 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0514 23:24:27.457219 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas W0514 23:24:35.808362 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0514 23:24:36.958171 3796 obj_retry.go:498] Detected object tsf/patch-tekton-config-z22jk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:36.958740 3796 obj_retry.go:498] Detected object tsf/patch-tekton-config-z22jk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:37.980370 3796 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:37.980394 3796 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:43.529890 3796 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-q77kq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:43.530396 3796 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-q77kq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:44.218450 3796 cni.go:452] [tsf-tas/rekor-createtree-job-7p2l9-j2vbg 184ebfb2904fdf8eec09b5e8edc7302d037172b8b064eabfd157601c950ba4d4 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-7p2l9-j2vbg 184ebfb2904fdf8eec09b5e8edc7302d037172b8b064eabfd157601c950ba4d4 network default NAD default NAD key ] I0514 23:24:44.308818 3796 cni.go:473] [tsf-tas/rekor-createtree-job-7p2l9-j2vbg 184ebfb2904fdf8eec09b5e8edc7302d037172b8b064eabfd157601c950ba4d4 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-7p2l9-j2vbg 184ebfb2904fdf8eec09b5e8edc7302d037172b8b064eabfd157601c950ba4d4 network default NAD default NAD key default], result "{}", err I0514 23:24:44.361071 3796 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:44.361101 3796 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-7p2l9-j2vbg I0514 23:24:44.362432 3796 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-7p2l9-j2vbg, ips: 10.129.0.37 I0514 23:24:44.362466 3796 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:44.549542 3796 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-q77kq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.549567 3796 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-q77kq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.583501 3796 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.583520 3796 obj_retry.go:492] Detected object tsf/patch-tekton-config-z22jk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:44.588704 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-z22jk I0514 23:24:44.588735 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-z22jk I0514 23:24:45.236820 3796 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:45.236841 3796 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-7p2l9-j2vbg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:48.545937 3796 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:48.546483 3796 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:48.565763 3796 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:48.565784 3796 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:48.572712 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0514 23:24:48.572736 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0514 23:24:49.926951 3796 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-kjtlv] creating logical port tsf-tpa_create-db-kjtlv for pod on switch ip-10-0-2-152.ec2.internal I0514 23:24:49.927058 3796 kube.go:256] Updating pod tsf-tpa/create-db-kjtlv I0514 23:24:49.941293 3796 pod.go:62] [tsf-tpa/create-db-kjtlv] pod update took 14.265666ms I0514 23:24:49.941316 3796 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-kjtlv] addLogicalPort annotation time took 14.293468ms I0514 23:24:49.942718 3796 pods.go:271] [tsf-tpa/create-db-kjtlv] addLogicalPort took 15.782072ms, libovsdb time 964.864µs I0514 23:24:50.326462 3796 cni.go:452] [tsf-tpa/create-db-kjtlv e856a0d1dcdb729bf8e067e579d5b956bb21a58f88e11cef0896416399673109 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-kjtlv e856a0d1dcdb729bf8e067e579d5b956bb21a58f88e11cef0896416399673109 network default NAD default NAD key ] I0514 23:24:50.336391 3796 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-kjtlv, hostIfaceName: e856a0d1dcdb729, network: default, NAD default, SandboxID: "e856a0d1dcdb729bf8e067e579d5b956bb21a58f88e11cef0896416399673109", PCI device ID: , UID: "f0199b9a-00a0-4d8d-b315-d38074e44964", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0514 23:24:50.391001 3796 cni.go:473] [tsf-tpa/create-db-kjtlv e856a0d1dcdb729bf8e067e579d5b956bb21a58f88e11cef0896416399673109 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-kjtlv e856a0d1dcdb729bf8e067e579d5b956bb21a58f88e11cef0896416399673109 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:36:dc:75:4b:fc\",\"name\":\"e856a0d1dcdb729\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/980fecdc-aa91-4478-b4ab-aa9dbf601ee4\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:24:52.899551 3796 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0514 23:24:52.920267 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0514 23:24:54.275540 3796 cni.go:452] [tsf-tpa/create-db-kjtlv e856a0d1dcdb729bf8e067e579d5b956bb21a58f88e11cef0896416399673109 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-kjtlv e856a0d1dcdb729bf8e067e579d5b956bb21a58f88e11cef0896416399673109 network default NAD default NAD key ] I0514 23:24:54.364153 3796 cni.go:473] [tsf-tpa/create-db-kjtlv e856a0d1dcdb729bf8e067e579d5b956bb21a58f88e11cef0896416399673109 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-kjtlv e856a0d1dcdb729bf8e067e579d5b956bb21a58f88e11cef0896416399673109 network default NAD default NAD key default], result "{}", err I0514 23:24:54.418822 3796 obj_retry.go:498] Detected object tsf-tpa/create-db-kjtlv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:54.418857 3796 pods.go:176] Deleting pod: tsf-tpa/create-db-kjtlv I0514 23:24:54.420405 3796 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-kjtlv, ips: 10.129.0.40 I0514 23:24:54.420438 3796 obj_retry.go:498] Detected object tsf-tpa/create-db-kjtlv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:55.441633 3796 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:55.441663 3796 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:55.545932 3796 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-4fs8n] creating logical port tsf-tpa_migrate-db-4fs8n for pod on switch ip-10-0-2-152.ec2.internal I0514 23:24:55.546066 3796 kube.go:256] Updating pod tsf-tpa/migrate-db-4fs8n I0514 23:24:55.559982 3796 pod.go:62] [tsf-tpa/migrate-db-4fs8n] pod update took 13.943626ms I0514 23:24:55.560006 3796 base_network_controller_pods.go:951] [default/tsf-tpa/migrate-db-4fs8n] addLogicalPort annotation time took 13.974646ms I0514 23:24:55.561324 3796 pods.go:271] [tsf-tpa/migrate-db-4fs8n] addLogicalPort took 15.407797ms, libovsdb time 887.814µs I0514 23:24:55.917065 3796 cni.go:452] [tsf-tpa/migrate-db-4fs8n 5383c9664f79952ff1e1b85bd0fafcefb224ef37f86ff183906d26f466a35a0a network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/migrate-db-4fs8n 5383c9664f79952ff1e1b85bd0fafcefb224ef37f86ff183906d26f466a35a0a network default NAD default NAD key ] I0514 23:24:55.927259 3796 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-4fs8n, hostIfaceName: 5383c9664f79952, network: default, NAD default, SandboxID: "5383c9664f79952ff1e1b85bd0fafcefb224ef37f86ff183906d26f466a35a0a", PCI device ID: , UID: "782f8965-5c44-418e-bdf2-fed3607d3e53", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0514 23:24:55.980628 3796 cni.go:473] [tsf-tpa/migrate-db-4fs8n 5383c9664f79952ff1e1b85bd0fafcefb224ef37f86ff183906d26f466a35a0a network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/migrate-db-4fs8n 5383c9664f79952ff1e1b85bd0fafcefb224ef37f86ff183906d26f466a35a0a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:81:f7:fa:55:ae\",\"name\":\"5383c9664f79952\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c2454584-6c48-4a1a-b8a2-43c85c119e31\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:24:58.302569 3796 cni.go:452] [tsf-tpa/migrate-db-4fs8n 5383c9664f79952ff1e1b85bd0fafcefb224ef37f86ff183906d26f466a35a0a network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/migrate-db-4fs8n 5383c9664f79952ff1e1b85bd0fafcefb224ef37f86ff183906d26f466a35a0a network default NAD default NAD key ] I0514 23:24:58.389883 3796 cni.go:473] [tsf-tpa/migrate-db-4fs8n 5383c9664f79952ff1e1b85bd0fafcefb224ef37f86ff183906d26f466a35a0a network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/migrate-db-4fs8n 5383c9664f79952ff1e1b85bd0fafcefb224ef37f86ff183906d26f466a35a0a network default NAD default NAD key default], result "{}", err I0514 23:24:58.444117 3796 obj_retry.go:498] Detected object tsf-tpa/migrate-db-4fs8n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:58.444152 3796 pods.go:176] Deleting pod: tsf-tpa/migrate-db-4fs8n I0514 23:24:58.445455 3796 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-4fs8n, ips: 10.129.0.41 I0514 23:24:58.445484 3796 obj_retry.go:498] Detected object tsf-tpa/migrate-db-4fs8n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:24:58.575646 3796 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:24:58.575677 3796 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:25:00.953681 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0514 23:25:01.705162 3796 base_network_controller_pods.go:487] [default/tsf-tpa/server-7b56d85679-gbgxv] creating logical port tsf-tpa_server-7b56d85679-gbgxv for pod on switch ip-10-0-2-152.ec2.internal I0514 23:25:01.705260 3796 kube.go:256] Updating pod tsf-tpa/server-7b56d85679-gbgxv I0514 23:25:01.721324 3796 pod.go:62] [tsf-tpa/server-7b56d85679-gbgxv] pod update took 16.091889ms I0514 23:25:01.721347 3796 base_network_controller_pods.go:951] [default/tsf-tpa/server-7b56d85679-gbgxv] addLogicalPort annotation time took 16.120718ms I0514 23:25:01.722597 3796 pods.go:271] [tsf-tpa/server-7b56d85679-gbgxv] addLogicalPort took 17.453191ms, libovsdb time 826.934µs I0514 23:25:03.023478 3796 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-xg9qr] creating logical port tsf-tpa_importer-86bcdf477b-xg9qr for pod on switch ip-10-0-2-152.ec2.internal I0514 23:25:03.023583 3796 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-xg9qr I0514 23:25:03.040937 3796 pod.go:62] [tsf-tpa/importer-86bcdf477b-xg9qr] pod update took 17.389342ms I0514 23:25:03.040958 3796 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-xg9qr] addLogicalPort annotation time took 17.4173ms I0514 23:25:03.042174 3796 pods.go:271] [tsf-tpa/importer-86bcdf477b-xg9qr] addLogicalPort took 18.712048ms, libovsdb time 821.534µs I0514 23:25:03.104981 3796 obj_retry.go:498] Detected object tsf-tpa/create-importers-x6kkn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:03.105527 3796 obj_retry.go:498] Detected object tsf-tpa/create-importers-x6kkn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:04.130167 3796 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:04.130194 3796 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:13.179916 3796 cni.go:452] [tsf-tpa/server-7b56d85679-gbgxv df504464801f72b314eb49e1f60c7345e77419695fedd1809f2d12655516ea33 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-7b56d85679-gbgxv df504464801f72b314eb49e1f60c7345e77419695fedd1809f2d12655516ea33 network default NAD default NAD key ] I0514 23:25:13.189444 3796 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-7b56d85679-gbgxv, hostIfaceName: df504464801f72b, network: default, NAD default, SandboxID: "df504464801f72b314eb49e1f60c7345e77419695fedd1809f2d12655516ea33", PCI device ID: , UID: "090ef279-7842-4a3a-b61a-23b2eef28459", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0514 23:25:13.246011 3796 cni.go:473] [tsf-tpa/server-7b56d85679-gbgxv df504464801f72b314eb49e1f60c7345e77419695fedd1809f2d12655516ea33 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-7b56d85679-gbgxv df504464801f72b314eb49e1f60c7345e77419695fedd1809f2d12655516ea33 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:ea:68:d0:32:3e\",\"name\":\"df504464801f72b\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d6ceb1cc-181a-415b-86a8-8464d8f1b451\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0514 23:25:13.288184 3796 cni.go:452] [tsf-tpa/importer-86bcdf477b-xg9qr d78d6929773b65d55bdfed35d67ca81492a338ba7e937c58b9ae6a03e7694dec network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-xg9qr d78d6929773b65d55bdfed35d67ca81492a338ba7e937c58b9ae6a03e7694dec network default NAD default NAD key ] I0514 23:25:13.298985 3796 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-xg9qr, hostIfaceName: d78d6929773b65d, network: default, NAD default, SandboxID: "d78d6929773b65d55bdfed35d67ca81492a338ba7e937c58b9ae6a03e7694dec", PCI device ID: , UID: "8b0e726b-9a35-4d2d-a190-294ca6560f4d", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0514 23:25:13.357201 3796 cni.go:473] [tsf-tpa/importer-86bcdf477b-xg9qr d78d6929773b65d55bdfed35d67ca81492a338ba7e937c58b9ae6a03e7694dec network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-xg9qr d78d6929773b65d55bdfed35d67ca81492a338ba7e937c58b9ae6a03e7694dec network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:78:72:fa:39:e4\",\"name\":\"d78d6929773b65d\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/edc7f049-f53c-4be2-a0c0-e06253da9098\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0514 23:25:14.416311 3796 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0514 23:25:15.346475 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=115 W0514 23:25:15.428365 3796 util.go:959] Failed to build node endpoints for node ip-10-0-2-152.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0514 23:25:16.721972 3796 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:16.722567 3796 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:16.743742 3796 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:16.743761 3796 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:16.750757 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0514 23:25:16.750790 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0514 23:25:17.483848 3796 namespace.go:98] [image-controller] adding namespace I0514 23:25:17.484737 3796 namespace.go:102] [image-controller] adding namespace took 869.039µs I0514 23:25:17.488373 3796 namespace.go:98] [default-tenant] adding namespace I0514 23:25:17.488425 3796 namespace.go:98] [integration-service] adding namespace I0514 23:25:17.488544 3796 namespace.go:98] [build-service] adding namespace I0514 23:25:17.489204 3796 namespace.go:102] [default-tenant] adding namespace took 808.516µs I0514 23:25:17.489846 3796 namespace.go:102] [integration-service] adding namespace took 1.408984ms I0514 23:25:17.490328 3796 namespace.go:102] [build-service] adding namespace took 1.769991ms I0514 23:25:17.513497 3796 namespace.go:142] [image-controller] updating namespace I0514 23:25:17.539199 3796 namespace.go:142] [image-controller] updating namespace I0514 23:25:17.547127 3796 namespace.go:142] [default-tenant] updating namespace I0514 23:25:17.582301 3796 namespace.go:142] [integration-service] updating namespace I0514 23:25:17.597209 3796 namespace.go:142] [default-tenant] updating namespace I0514 23:25:17.599932 3796 namespace.go:142] [integration-service] updating namespace I0514 23:25:17.609347 3796 namespace.go:142] [build-service] updating namespace I0514 23:25:17.629918 3796 namespace.go:142] [build-service] updating namespace I0514 23:25:17.757453 3796 namespace.go:142] [build-service] updating namespace I0514 23:25:17.817704 3796 namespace.go:142] [integration-service] updating namespace I0514 23:25:17.869245 3796 namespace.go:98] [release-service] adding namespace I0514 23:25:17.870050 3796 namespace.go:102] [release-service] adding namespace took 780.928µs I0514 23:25:17.900207 3796 namespace.go:142] [release-service] updating namespace I0514 23:25:17.921047 3796 namespace.go:142] [release-service] updating namespace I0514 23:25:18.036129 3796 namespace.go:142] [konflux-ui] updating namespace I0514 23:25:18.171250 3796 namespace.go:98] [namespace-lister] adding namespace I0514 23:25:18.171318 3796 namespace.go:98] [konflux-info] adding namespace I0514 23:25:18.172136 3796 namespace.go:102] [namespace-lister] adding namespace took 864.072µs I0514 23:25:18.172691 3796 namespace.go:102] [konflux-info] adding namespace took 1.357359ms I0514 23:25:18.266061 3796 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:18.266134 3796 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:18.266308 3796 namespace.go:142] [konflux-info] updating namespace I0514 23:25:18.297049 3796 namespace.go:142] [konflux-info] updating namespace I0514 23:25:18.421124 3796 namespace.go:98] [enterprise-contract-service] adding namespace I0514 23:25:18.422079 3796 namespace.go:102] [enterprise-contract-service] adding namespace took 925.028µs I0514 23:25:18.422122 3796 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:18.434852 3796 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:18.434938 3796 namespace.go:142] [image-controller] updating namespace I0514 23:25:19.036165 3796 namespace.go:142] [default-tenant] updating namespace I0514 23:25:19.655250 3796 namespace.go:98] [konflux-cli] adding namespace I0514 23:25:19.656213 3796 namespace.go:102] [konflux-cli] adding namespace took 944.762µs I0514 23:25:19.656262 3796 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:19.680356 3796 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:19.941253 3796 namespace.go:98] [segment-bridge] adding namespace I0514 23:25:19.942114 3796 namespace.go:102] [segment-bridge] adding namespace took 839.107µs I0514 23:25:20.059174 3796 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:20.074249 3796 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:20.264418 3796 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0514 23:25:20.265072 3796 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0514 23:25:20.265089 3796 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0514 23:25:20.268119 3796 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0514 23:25:20.444351 3796 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0514 23:25:20.444381 3796 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0514 23:25:20.445272 3796 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0514 23:25:21.032263 3796 namespace.go:142] [build-service] updating namespace I0514 23:25:21.795403 3796 namespace.go:142] [default-tenant] updating namespace I0514 23:25:21.974512 3796 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:22.093832 3796 namespace.go:142] [image-controller] updating namespace I0514 23:25:23.026028 3796 namespace.go:142] [integration-service] updating namespace I0514 23:25:23.242623 3796 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:23.488026 3796 namespace.go:142] [konflux-info] updating namespace I0514 23:25:23.742582 3796 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:23.923652 3796 namespace.go:142] [release-service] updating namespace I0514 23:25:24.052202 3796 namespace.go:142] [build-service] updating namespace I0514 23:25:24.073900 3796 namespace.go:142] [default-tenant] updating namespace I0514 23:25:24.090553 3796 namespace.go:142] [enterprise-contract-service] updating namespace W0514 23:25:24.102230 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0514 23:25:24.104381 3796 namespace.go:142] [image-controller] updating namespace I0514 23:25:24.120090 3796 namespace.go:142] [integration-service] updating namespace I0514 23:25:24.164850 3796 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:24.183545 3796 namespace.go:142] [konflux-info] updating namespace I0514 23:25:24.208448 3796 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:24.222347 3796 namespace.go:142] [release-service] updating namespace I0514 23:25:24.280894 3796 namespace.go:142] [build-service] updating namespace I0514 23:25:24.340967 3796 namespace.go:142] [default-tenant] updating namespace I0514 23:25:24.411106 3796 namespace.go:142] [enterprise-contract-service] updating namespace I0514 23:25:24.475986 3796 namespace.go:142] [image-controller] updating namespace I0514 23:25:24.576116 3796 namespace.go:142] [integration-service] updating namespace I0514 23:25:24.676141 3796 namespace.go:142] [konflux-cli] updating namespace I0514 23:25:24.778362 3796 namespace.go:142] [konflux-info] updating namespace I0514 23:25:24.880120 3796 namespace.go:142] [namespace-lister] updating namespace I0514 23:25:24.969003 3796 namespace.go:142] [release-service] updating namespace I0514 23:25:26.224851 3796 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:26.355757 3796 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:26.458024 3796 namespace.go:142] [segment-bridge] updating namespace I0514 23:25:26.831117 3796 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:26.831817 3796 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0514 23:25:27.674793 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0514 23:25:27.852921 3796 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:27.852945 3796 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-s6cwq-rwlz5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0514 23:25:28.116837 3796 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0514 23:25:31.136242 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0514 23:25:31.138904 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0514 23:25:32.714099 3796 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0514 23:25:33.721334 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0514 23:25:33.723698 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0514 23:25:36.176227 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:36.176246 3796 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:38.779848 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0514 23:25:38.780118 3796 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0514 23:25:38.799955 3796 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0514 23:25:38.839439 3796 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0514 23:25:38.839454 3796 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0514 23:25:42.498829 3796 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:42.499500 3796 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:25:42.792651 3796 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:42.792679 3796 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:25:42.796929 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x I0514 23:25:42.796957 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-vkt8x I0514 23:25:44.464902 3796 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0514 23:26:05.471710 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0514 23:26:45.431481 3796 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0514 23:26:48.357060 3796 obj_retry.go:498] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:48.357679 3796 obj_retry.go:498] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:48.589393 3796 obj_retry.go:492] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:48.589416 3796 obj_retry.go:492] Detected object konflux-ui/proxy-78867c4458-c44l9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:48.597296 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-78867c4458-c44l9 I0514 23:26:48.597324 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-78867c4458-c44l9 I0514 23:26:49.734068 3796 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:49.734709 3796 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:26:49.778289 3796 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:49.778309 3796 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:26:49.785343 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0514 23:26:49.785368 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0514 23:27:09.563800 3796 namespace.go:98] [managed-tenant-hjy2g] adding namespace I0514 23:27:09.564817 3796 namespace.go:102] [managed-tenant-hjy2g] adding namespace took 990.652µs I0514 23:27:09.589308 3796 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:09.608919 3796 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:10.617847 3796 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:10.751581 3796 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:10.852411 3796 namespace.go:142] [managed-tenant-hjy2g] updating namespace I0514 23:27:27.435675 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0514 23:28:27.495508 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0514 23:28:48.345932 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=67 I0514 23:29:08.345264 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0514 23:30:04.464655 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0514 23:30:04.465310 3796 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0514 23:30:05.329894 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:30:05.329948 3796 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29646690-92lfd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:30:07.447359 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0514 23:30:11.430950 3796 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 I0514 23:30:16.459102 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0514 23:30:26.348336 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=80 I0514 23:30:43.346485 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=47 I0514 23:30:50.349950 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=50 I0514 23:32:04.360911 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=347 I0514 23:32:12.454667 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0514 23:33:05.439216 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=5 I0514 23:33:46.434438 3796 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0514 23:33:51.441088 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0514 23:34:21.474748 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0514 23:34:49.468180 3796 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=11 I0514 23:34:55.051526 3796 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:55.051569 3796 obj_retry.go:492] Detected object tsf-tpa/create-db-kjtlv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:55.057511 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-kjtlv I0514 23:34:55.057548 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-kjtlv I0514 23:34:58.041448 3796 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:58.041503 3796 obj_retry.go:492] Detected object tsf-tpa/migrate-db-4fs8n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:34:58.047242 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-4fs8n I0514 23:34:58.047271 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-4fs8n I0514 23:35:04.032555 3796 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:35:04.032615 3796 obj_retry.go:492] Detected object tsf-tpa/create-importers-x6kkn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0514 23:35:04.037106 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-x6kkn I0514 23:35:04.037127 3796 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-x6kkn I0514 23:36:07.349048 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=41 I0514 23:36:40.434799 3796 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0514 23:36:41.461885 3796 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0514 23:37:09.347966 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0514 23:37:43.352274 3796 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8