+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-48.ec2.internal ++ [[ -n ip-10-0-0-48.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-48.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 22:58:53.502436903 - disable conntrack on geneve port' I0327 22:58:53.502436903 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0327 22:58:53.518724229 - starting ovnkube-node + echo 'I0327 22:58:53.518724229 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-48.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-48.ec2.internal --init-node ip-10-0-0-48.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-48.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0327 22:58:53.556717 3898 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0327 22:58:53.556779 3898 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-29bd08d30b.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0327 22:58:53.558514 3898 kube.go:419] Waiting for certificate I0327 22:58:53.558545 3898 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0327 22:58:53.558576 3898 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0327 22:58:53.559283 3898 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 22:58:53.587521 3898 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 22:58:53.587539 3898 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 22:58:53.590502 3898 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 22:58:53.593650 3898 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-kbwnm" I0327 22:58:53.600568 3898 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-kbwnm" I0327 22:58:53.600633 3898 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 22:58:54.558864 3898 kube.go:426] Certificate found I0327 22:58:54.559488 3898 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 22:58:54.560338 3898 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0327 22:58:54.604654 3898 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-28 22:53:53 +0000 UTC" deadline="2026-03-28 18:50:52.087407816 +0000 UTC" I0327 22:58:54.604705 3898 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h51m57.482717225s" I0327 22:58:54.623529 3898 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 22:58:54.630945 3898 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0327 22:58:54.632016 3898 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0327 22:58:54.632081 3898 factory.go:531] Starting watch factory I0327 22:58:54.632163 3898 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632177 3898 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632190 3898 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632207 3898 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632223 3898 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632196 3898 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632224 3898 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632269 3898 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632503 3898 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632543 3898 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.632977 3898 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.633165 3898 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.639012 3898 controller_manager.go:368] Starting the ovnkube controller I0327 22:58:54.639038 3898 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-48.ec2.internal I0327 22:58:54.639107 3898 controller_manager.go:393] NBDB zone sync took: 59.643µs I0327 22:58:54.639119 3898 factory.go:531] Starting watch factory I0327 22:58:54.656239 3898 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.658563 3898 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.658940 3898 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.659030 3898 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.659386 3898 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0327 22:58:54.659674 3898 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.659968 3898 ovn_northd.go:121] Registering OVN northd metrics I0327 22:58:54.663734 3898 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0327 22:58:54.663805 3898 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0327 22:58:54.670680 3898 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 22:58:54.679060 3898 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0327 22:58:54.738178 3898 factory.go:1890] *v1.Node informer cache synced successfully I0327 22:58:54.738194 3898 factory.go:1890] *v1.Service informer cache synced successfully I0327 22:58:54.738197 3898 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 22:58:54.738201 3898 factory.go:1890] *v1.Pod informer cache synced successfully I0327 22:58:54.738204 3898 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 22:58:54.738207 3898 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 22:58:54.738290 3898 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:54.738285 3898 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:54.738306 3898 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:54.738348 3898 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:54.740688 3898 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:54.740691 3898 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 22:58:54.743587 3898 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 22:58:54.743599 3898 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 22:58:54.743650 3898 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.743664 3898 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.745547 3898 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.749325 3898 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 22:58:54.749339 3898 factory.go:1890] *v1.Pod informer cache synced successfully I0327 22:58:54.749346 3898 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 22:58:54.749350 3898 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 22:58:54.749353 3898 factory.go:1890] *v1.Node informer cache synced successfully I0327 22:58:54.749357 3898 factory.go:1890] *v1.Service informer cache synced successfully I0327 22:58:54.749360 3898 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 22:58:54.749383 3898 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.749395 3898 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.751486 3898 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.755195 3898 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 22:58:54.755205 3898 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 22:58:54.755205 3898 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 22:58:54.755256 3898 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.755267 3898 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.757220 3898 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.760855 3898 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 22:58:54.760913 3898 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.760927 3898 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.761028 3898 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 22:58:54.762760 3898 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.766402 3898 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 22:58:54.766407 3898 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 22:58:54.766479 3898 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.766491 3898 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.768482 3898 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.772281 3898 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 22:58:54.772292 3898 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 22:58:54.772480 3898 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 22:58:54.772496 3898 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 22:58:54.774624 3898 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 22:58:54.777383 3898 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 22:58:54.777395 3898 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 22:58:54.777452 3898 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.777477 3898 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.777479 3898 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.777501 3898 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.779686 3898 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.779690 3898 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 22:58:54.782824 3898 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 22:58:54.782836 3898 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 22:58:54.782842 3898 factory.go:636] Watch Factory start up complete, took: 150.763951ms I0327 22:58:54.782914 3898 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0327 22:58:54.782923 3898 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 22:58:54.788092 3898 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 22:58:54.793177 3898 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 22:58:54.793186 3898 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 22:58:54.793192 3898 factory.go:636] Watch Factory start up complete, took: 154.072885ms I0327 22:58:54.793200 3898 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-0-48.ec2.internal" zone I0327 22:58:54.817046 3898 default_node_network_controller.go:709] Initializing the default node network controller I0327 22:58:54.868308 3898 udn_isolation.go:101] Starting UDN host isolation manager I0327 22:58:54.874499 3898 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0327 22:58:54.920252 3898 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0327 22:58:54.920322 3898 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0327 22:58:54.920335 3898 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0327 22:58:54.952090 3898 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0327 22:58:54.952151 3898 default_node_network_controller.go:808] Node ip-10-0-0-48.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0327 22:58:54.957254 3898 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.48"] k8s.ovn.org/zone-name:ip-10-0-0-48.ec2.internal] on node ip-10-0-0-48.ec2.internal I0327 22:58:54.977042 3898 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0327 22:58:54.982151 3898 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0327 22:58:54.982363 3898 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0327 22:58:54.982526 3898 gateway_init.go:263] Preparing Gateway I0327 22:58:54.982542 3898 gateway_shared_intf.go:1658] Creating new gateway I0327 22:58:55.043251 3898 controller_manager.go:427] Waiting for node in zone sync took: 250.03882ms I0327 22:58:55.047228 3898 controller_manager.go:608] Switching to transit router for layer2 networks I0327 22:58:55.047243 3898 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-48.ec2.internal I0327 22:58:55.054677 3898 iptables.go:108] Creating table: filter chain: FORWARD I0327 22:58:55.060810 3898 iptables.go:108] Creating table: filter chain: OUTPUT I0327 22:58:55.061617 3898 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 22:58:55.066975 3898 controller_manager.go:288] SCTP support detected in OVN I0327 22:58:55.067104 3898 gateway_shared_intf.go:1787] Gateway Creation Complete I0327 22:58:55.067118 3898 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-48.ec2.internal","mac-address":"12:34:ec:aa:24:9b","ip-addresses":["10.0.0.48/24"],"ip-address":"10.0.0.48/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:63d81019-9a7c-45e3-a948-15f62537fa6e k8s.ovn.org/node-encap-ips:["10.0.0.48"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.48/24"} k8s.ovn.org/zone-name:ip-10-0-0-48.ec2.internal] on node ip-10-0-0-48.ec2.internal I0327 22:58:55.067544 3898 services_controller.go:69] Creating services controller for network=default I0327 22:58:55.067649 3898 udn_enabled_svc.go:80] Waiting for service informer to sync I0327 22:58:55.067735 3898 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0327 22:58:55.067765 3898 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0327 22:58:55.067778 3898 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0327 22:58:55.067786 3898 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 22:58:55.067932 3898 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0327 22:58:55.067980 3898 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0327 22:58:55.068005 3898 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 43.362µs I0327 22:58:55.068014 3898 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 24.853µs I0327 22:58:55.068023 3898 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 41.58µs I0327 22:58:55.068030 3898 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0327 22:58:55.068044 3898 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0327 22:58:55.068056 3898 default_network_controller.go:321] Starting the default network controller I0327 22:58:55.068086 3898 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 23.384µs I0327 22:58:55.068202 3898 udn_enabled_svc.go:95] Performing full resync I0327 22:58:55.068555 3898 udn_enabled_svc.go:99] Waiting for handler to sync I0327 22:58:55.068568 3898 udn_enabled_svc.go:104] Starting worker I0327 22:58:55.076374 3898 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0327 22:58:55.076430 3898 repair.go:33] Syncing exgw routes took 43.832µs I0327 22:58:55.076454 3898 default_network_controller.go:407] Starting all the Watchers... I0327 22:58:55.076673 3898 namespace.go:98] [openshift-backplane-cee] adding namespace I0327 22:58:55.076679 3898 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0327 22:58:55.076687 3898 namespace.go:98] [openshift-backplane-srep] adding namespace I0327 22:58:55.076696 3898 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0327 22:58:55.076697 3898 namespace.go:98] [openshift-infra] adding namespace I0327 22:58:55.076706 3898 namespace.go:98] [openshift-kube-apiserver] adding namespace I0327 22:58:55.076709 3898 namespace.go:98] [openshift-backplane] adding namespace I0327 22:58:55.076718 3898 namespace.go:98] [kube-system] adding namespace I0327 22:58:55.076721 3898 namespace.go:98] [openshift-ingress] adding namespace I0327 22:58:55.076721 3898 namespace.go:98] [openshift-authentication] adding namespace I0327 22:58:55.076742 3898 namespace.go:98] [openshift-backplane-csm] adding namespace I0327 22:58:55.076755 3898 namespace.go:98] [openshift-config-operator] adding namespace I0327 22:58:55.076758 3898 namespace.go:98] [openshift-apiserver] adding namespace I0327 22:58:55.076784 3898 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0327 22:58:55.076676 3898 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0327 22:58:55.077312 3898 namespace.go:102] [openshift-backplane-cee] adding namespace took 624.076µs I0327 22:58:55.077327 3898 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0327 22:58:55.077801 3898 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 467.122µs I0327 22:58:55.077820 3898 namespace.go:98] [openshift-etcd] adding namespace I0327 22:58:55.078293 3898 namespace.go:102] [openshift-etcd] adding namespace took 466.317µs I0327 22:58:55.078306 3898 namespace.go:98] [openshift-config-managed] adding namespace I0327 22:58:55.079372 3898 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 2.683453ms I0327 22:58:55.079385 3898 namespace.go:98] [openshift-must-gather-operator] adding namespace I0327 22:58:55.079825 3898 namespace.go:102] [openshift-backplane-srep] adding namespace took 3.129338ms I0327 22:58:55.079841 3898 namespace.go:98] [openshift-route-controller-manager] adding namespace I0327 22:58:55.080298 3898 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 3.597077ms I0327 22:58:55.080312 3898 namespace.go:98] [openshift-machine-api] adding namespace I0327 22:58:55.080727 3898 namespace.go:102] [openshift-infra] adding namespace took 4.021095ms I0327 22:58:55.080738 3898 namespace.go:98] [openshift-ingress-canary] adding namespace I0327 22:58:55.081149 3898 namespace.go:102] [openshift-kube-apiserver] adding namespace took 4.436951ms I0327 22:58:55.081174 3898 namespace.go:98] [openshift-package-operator] adding namespace I0327 22:58:55.081618 3898 namespace.go:102] [openshift-backplane] adding namespace took 4.904657ms I0327 22:58:55.081631 3898 namespace.go:98] [openshift-console-user-settings] adding namespace I0327 22:58:55.082086 3898 namespace.go:102] [kube-system] adding namespace took 5.361568ms I0327 22:58:55.082102 3898 namespace.go:98] [openshift-dns-operator] adding namespace I0327 22:58:55.082490 3898 namespace.go:102] [openshift-ingress] adding namespace took 5.76388ms I0327 22:58:55.082503 3898 namespace.go:98] [openshift-authentication-operator] adding namespace I0327 22:58:55.082937 3898 namespace.go:102] [openshift-authentication] adding namespace took 6.203498ms I0327 22:58:55.082954 3898 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0327 22:58:55.083424 3898 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.668106ms I0327 22:58:55.083437 3898 namespace.go:98] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] adding namespace I0327 22:58:55.083879 3898 namespace.go:102] [openshift-config-operator] adding namespace took 7.113399ms I0327 22:58:55.083894 3898 namespace.go:98] [openshift-customer-monitoring] adding namespace I0327 22:58:55.084273 3898 namespace.go:102] [openshift-apiserver] adding namespace took 7.505648ms I0327 22:58:55.084283 3898 namespace.go:98] [openshift] adding namespace I0327 22:58:55.084695 3898 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.901466ms I0327 22:58:55.084708 3898 namespace.go:98] [openshift-kube-scheduler] adding namespace I0327 22:58:55.085230 3898 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 8.432775ms I0327 22:58:55.085246 3898 namespace.go:98] [openshift-host-network] adding namespace I0327 22:58:55.085629 3898 namespace.go:102] [openshift-config-managed] adding namespace took 7.318402ms I0327 22:58:55.085641 3898 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0327 22:58:55.086016 3898 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.626091ms I0327 22:58:55.086030 3898 namespace.go:98] [openshift-backplane-mobb] adding namespace I0327 22:58:55.086382 3898 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.536863ms I0327 22:58:55.086393 3898 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0327 22:58:55.086797 3898 namespace.go:102] [openshift-machine-api] adding namespace took 6.480711ms I0327 22:58:55.086806 3898 namespace.go:98] [openshift-apiserver-operator] adding namespace I0327 22:58:55.087442 3898 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.698505ms I0327 22:58:55.087459 3898 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0327 22:58:55.088010 3898 namespace.go:102] [openshift-package-operator] adding namespace took 6.829162ms I0327 22:58:55.088024 3898 namespace.go:98] [openshift-node] adding namespace I0327 22:58:55.088406 3898 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.767394ms I0327 22:58:55.088422 3898 namespace.go:98] [openshift-marketplace] adding namespace I0327 22:58:55.088890 3898 namespace.go:102] [openshift-dns-operator] adding namespace took 6.781986ms I0327 22:58:55.088908 3898 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0327 22:58:55.089350 3898 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.843191ms I0327 22:58:55.089360 3898 namespace.go:98] [openshift-dns] adding namespace I0327 22:58:55.089880 3898 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.91875ms I0327 22:58:55.089896 3898 default_node_network_controller.go:518] Node connection status = connected I0327 22:58:55.089930 3898 namespace.go:98] [openshift-image-registry] adding namespace I0327 22:58:55.090423 3898 namespace.go:102] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] adding namespace took 6.980805ms I0327 22:58:55.090437 3898 namespace.go:98] [openshift-config] adding namespace I0327 22:58:55.090885 3898 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.986096ms I0327 22:58:55.090899 3898 namespace.go:98] [openshift-cluster-version] adding namespace I0327 22:58:55.091321 3898 namespace.go:102] [openshift] adding namespace took 7.032326ms I0327 22:58:55.091333 3898 namespace.go:98] [openshift-network-diagnostics] adding namespace I0327 22:58:55.091752 3898 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.039324ms I0327 22:58:55.091771 3898 namespace.go:98] [openshift-logging] adding namespace I0327 22:58:55.092249 3898 namespace.go:102] [openshift-host-network] adding namespace took 6.996802ms I0327 22:58:55.092263 3898 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0327 22:58:55.092694 3898 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.048455ms I0327 22:58:55.093076 3898 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.040225ms I0327 22:58:55.093088 3898 namespace.go:98] [openshift-service-ca-operator] adding namespace I0327 22:58:55.093480 3898 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.08296ms I0327 22:58:55.093491 3898 namespace.go:98] [default] adding namespace I0327 22:58:55.093912 3898 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.100528ms I0327 22:58:55.093925 3898 namespace.go:98] [openshift-backplane-tam] adding namespace I0327 22:58:55.094297 3898 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.81972ms I0327 22:58:55.094315 3898 namespace.go:98] [kube-public] adding namespace I0327 22:58:55.094697 3898 namespace.go:102] [openshift-node] adding namespace took 6.669115ms I0327 22:58:55.094708 3898 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0327 22:58:55.095046 3898 namespace.go:102] [openshift-marketplace] adding namespace took 6.617469ms I0327 22:58:55.095067 3898 namespace.go:98] [openshift-machine-config-operator] adding namespace I0327 22:58:55.095509 3898 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.594122ms I0327 22:58:55.095527 3898 namespace.go:98] [openshift-console-operator] adding namespace I0327 22:58:55.096031 3898 namespace.go:102] [openshift-dns] adding namespace took 6.661738ms I0327 22:58:55.096048 3898 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0327 22:58:55.096493 3898 namespace.go:102] [openshift-image-registry] adding namespace took 6.553082ms I0327 22:58:55.096510 3898 namespace.go:98] [openshift-insights] adding namespace I0327 22:58:55.096998 3898 namespace.go:102] [openshift-config] adding namespace took 6.553213ms I0327 22:58:55.097016 3898 namespace.go:98] [openshift-network-console] adding namespace W0327 22:58:55.097410 3898 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-qgvsg: pod openshift-network-diagnostics/network-check-target-qgvsg: no pod IPs found I0327 22:58:55.097436 3898 namespace.go:102] [openshift-cluster-version] adding namespace took 6.531304ms I0327 22:58:55.097447 3898 namespace.go:98] [openshift-monitoring] adding namespace I0327 22:58:55.097895 3898 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.556724ms I0327 22:58:55.097907 3898 namespace.go:98] [dedicated-admin] adding namespace I0327 22:58:55.098295 3898 namespace.go:102] [openshift-logging] adding namespace took 6.518614ms I0327 22:58:55.098307 3898 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0327 22:58:55.098670 3898 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.400322ms I0327 22:58:55.098694 3898 namespace.go:98] [kube-node-lease] adding namespace I0327 22:58:55.099110 3898 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.01532ms I0327 22:58:55.099127 3898 namespace.go:98] [openshift-ingress-operator] adding namespace I0327 22:58:55.099658 3898 namespace.go:102] [default] adding namespace took 6.161054ms I0327 22:58:55.099673 3898 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0327 22:58:55.100048 3898 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.118948ms I0327 22:58:55.100059 3898 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0327 22:58:55.100489 3898 namespace.go:102] [kube-public] adding namespace took 6.168741ms I0327 22:58:55.100501 3898 namespace.go:98] [openshift-operators-redhat] adding namespace I0327 22:58:55.101266 3898 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.553231ms I0327 22:58:55.101280 3898 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0327 22:58:55.101684 3898 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.612212ms I0327 22:58:55.101698 3898 namespace.go:98] [openshift-multus] adding namespace I0327 22:58:55.102071 3898 namespace.go:102] [openshift-console-operator] adding namespace took 6.538498ms I0327 22:58:55.102083 3898 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0327 22:58:55.102454 3898 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.400811ms I0327 22:58:55.102482 3898 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0327 22:58:55.102846 3898 namespace.go:102] [openshift-insights] adding namespace took 6.329999ms I0327 22:58:55.102858 3898 namespace.go:98] [openshift-network-operator] adding namespace I0327 22:58:55.103218 3898 namespace.go:102] [openshift-network-console] adding namespace took 6.197015ms I0327 22:58:55.103229 3898 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0327 22:58:55.103622 3898 namespace.go:102] [openshift-monitoring] adding namespace took 6.170124ms I0327 22:58:55.103635 3898 namespace.go:98] [openshift-backplane-cse] adding namespace I0327 22:58:55.104002 3898 namespace.go:102] [dedicated-admin] adding namespace took 6.090865ms I0327 22:58:55.104014 3898 namespace.go:98] [openshift-controller-manager] adding namespace I0327 22:58:55.104391 3898 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.080238ms I0327 22:58:55.104403 3898 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0327 22:58:55.104808 3898 namespace.go:102] [kube-node-lease] adding namespace took 6.106714ms I0327 22:58:55.104824 3898 namespace.go:98] [openshift-console] adding namespace I0327 22:58:55.105184 3898 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.053287ms I0327 22:58:55.105602 3898 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 5.92356ms I0327 22:58:55.105960 3898 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 5.895435ms I0327 22:58:55.106374 3898 namespace.go:102] [openshift-operators-redhat] adding namespace took 5.868476ms I0327 22:58:55.106386 3898 namespace.go:98] [openshift-kube-controller-manager] adding namespace W0327 22:58:55.106757 3898 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-n7l7j: pod openshift-multus/network-metrics-daemon-n7l7j: no pod IPs found I0327 22:58:55.106786 3898 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 5.501841ms I0327 22:58:55.106797 3898 namespace.go:98] [openshift-network-node-identity] adding namespace I0327 22:58:55.107219 3898 namespace.go:102] [openshift-multus] adding namespace took 5.515759ms I0327 22:58:55.107605 3898 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.517389ms I0327 22:58:55.107618 3898 namespace.go:98] [openshift-operators] adding namespace I0327 22:58:55.108002 3898 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.514295ms I0327 22:58:55.108365 3898 namespace.go:102] [openshift-network-operator] adding namespace took 5.502068ms I0327 22:58:55.108748 3898 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 5.514237ms I0327 22:58:55.108762 3898 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0327 22:58:55.109102 3898 namespace.go:102] [openshift-backplane-cse] adding namespace took 5.462297ms I0327 22:58:55.109484 3898 namespace.go:102] [openshift-controller-manager] adding namespace took 5.463653ms I0327 22:58:55.109858 3898 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.44993ms I0327 22:58:55.110223 3898 namespace.go:102] [openshift-console] adding namespace took 5.393953ms I0327 22:58:55.110647 3898 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 4.254844ms I0327 22:58:55.111031 3898 namespace.go:102] [openshift-network-node-identity] adding namespace took 4.228632ms I0327 22:58:55.111406 3898 namespace.go:102] [openshift-operators] adding namespace took 3.78326ms I0327 22:58:55.111757 3898 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 2.990026ms I0327 22:58:55.112117 3898 zone_ic_handler.go:186] Time taken to create transit switch: 250.607µs I0327 22:58:55.112196 3898 master.go:532] Adding or Updating local node "ip-10-0-0-48.ec2.internal" for network "default" I0327 22:58:55.112712 3898 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-88.ec2.internal for the network default I0327 22:58:55.112842 3898 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-195.ec2.internal for the network default I0327 22:58:55.113694 3898 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-88.ec2.internal" on network "default" took: 964.059µs I0327 22:58:55.113815 3898 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-195.ec2.internal" on network "default" took: 980.228µs W0327 22:58:55.119424 3898 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-n7l7j: pod openshift-multus/network-metrics-daemon-n7l7j: no pod IPs found W0327 22:58:55.119438 3898 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-qgvsg: pod openshift-network-diagnostics/network-check-target-qgvsg: no pod IPs found I0327 22:58:55.120344 3898 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-0-48.ec2.internal for the network default I0327 22:58:55.121478 3898 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0327 22:58:55.121512 3898 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0327 22:58:55.121529 3898 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0327 22:58:55.121533 3898 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-48.ec2.internal in network "default" I0327 22:58:55.121585 3898 node_tracker.go:169] Node ip-10-0-0-48.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:55.121598 3898 services_controller.go:574] Full service sync requested for network=default I0327 22:58:55.121604 3898 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-88.ec2.internal in network "default" I0327 22:58:55.121626 3898 node_tracker.go:169] Node ip-10-0-1-88.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:55.121633 3898 services_controller.go:574] Full service sync requested for network=default I0327 22:58:55.121640 3898 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-195.ec2.internal in network "default" I0327 22:58:55.121660 3898 node_tracker.go:169] Node ip-10-0-2-195.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:55.121666 3898 services_controller.go:574] Full service sync requested for network=default I0327 22:58:55.222398 3898 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0327 22:58:55.222422 3898 services_controller.go:208] Setting up event handlers for services for network=default I0327 22:58:55.222486 3898 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0327 22:58:55.222520 3898 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0327 22:58:55.222531 3898 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0327 22:58:55.323320 3898 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0327 22:58:55.323339 3898 repair.go:60] Starting repairing loop for services I0327 22:58:55.323434 3898 repair.go:131] Deleted 0 stale service LBs I0327 22:58:55.323453 3898 repair.go:137] Deleted 0 stale Chassis Template Vars I0327 22:58:55.323482 3898 repair.go:62] Finished repairing loop for services: 143.748µs I0327 22:58:55.323512 3898 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0327 22:58:55.323522 3898 services_controller.go:254] Starting workers for network=default I0327 22:58:55.324133 3898 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-qgvsg] creating logical port openshift-network-diagnostics_network-check-target-qgvsg for pod on switch ip-10-0-0-48.ec2.internal I0327 22:58:55.324225 3898 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-n7l7j] creating logical port openshift-multus_network-metrics-daemon-n7l7j for pod on switch ip-10-0-0-48.ec2.internal I0327 22:58:55.324247 3898 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-qgvsg I0327 22:58:55.324339 3898 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-n7l7j I0327 22:58:55.324726 3898 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-n9kkh of type *v1.Pod in terminal state (e.g. completed) during add event: will remove it W0327 22:58:55.327958 3898 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 22:58:55.349431 3898 pod.go:62] [openshift-multus/network-metrics-daemon-n7l7j] pod update took 25.134682ms I0327 22:58:55.349455 3898 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-n7l7j] addLogicalPort annotation time took 25.155489ms I0327 22:58:55.349431 3898 pod.go:62] [openshift-network-diagnostics/network-check-target-qgvsg] pod update took 25.225703ms I0327 22:58:55.349673 3898 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-qgvsg] addLogicalPort annotation time took 25.468481ms I0327 22:58:55.350598 3898 pods.go:271] [openshift-multus/network-metrics-daemon-n7l7j] addLogicalPort took 26.377867ms, libovsdb time 872.689µs I0327 22:58:55.350969 3898 pods.go:271] [openshift-network-diagnostics/network-check-target-qgvsg] addLogicalPort took 26.851076ms, libovsdb time 1.036859ms I0327 22:58:55.351581 3898 admin_network_policy_controller.go:227] Starting controller default-network-controller I0327 22:58:55.351606 3898 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0327 22:58:55.351625 3898 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0327 22:58:55.351631 3898 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0327 22:58:55.351801 3898 repair.go:29] Repairing admin network policies took 164.417µs I0327 22:58:55.351898 3898 repair.go:92] Repairing baseline admin network policies took 87.814µs I0327 22:58:55.352240 3898 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0327 22:58:55.352262 3898 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0327 22:58:55.354337 3898 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0327 22:58:55.354357 3898 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0327 22:58:55.354365 3898 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0327 22:58:55.355667 3898 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0327 22:58:55.355684 3898 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0327 22:58:55.355695 3898 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0327 22:58:55.356585 3898 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0327 22:58:55.356605 3898 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0327 22:58:55.356617 3898 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0327 22:58:55.358774 3898 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0327 22:58:55.358792 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0327 22:58:55.358799 3898 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0327 22:58:55.359958 3898 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0327 22:58:55.359977 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0327 22:58:55.359985 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0327 22:58:55.361794 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0327 22:58:55.361812 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0327 22:58:55.362602 3898 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0327 22:58:55.362615 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 22:58:55.363417 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0327 22:58:55.363436 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0327 22:58:55.363444 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0327 22:58:55.364873 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0327 22:58:55.364892 3898 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0327 22:58:55.364902 3898 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0327 22:58:55.365366 3898 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0327 22:58:55.365379 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0327 22:58:55.365387 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0327 22:58:55.365841 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0327 22:58:55.365860 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0327 22:58:55.365869 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0327 22:58:55.366320 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0327 22:58:55.366338 3898 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0327 22:58:55.366348 3898 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0327 22:58:55.367484 3898 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0327 22:58:55.367501 3898 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0327 22:58:55.367509 3898 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0327 22:58:55.368949 3898 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0327 22:58:55.368965 3898 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0327 22:58:55.368975 3898 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0327 22:58:55.370083 3898 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0327 22:58:55.370103 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0327 22:58:55.370130 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 22:58:55.370771 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0327 22:58:55.370793 3898 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0327 22:58:55.370800 3898 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0327 22:58:55.371286 3898 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0327 22:58:55.371303 3898 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0327 22:58:55.371310 3898 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0327 22:58:55.371672 3898 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0327 22:58:55.371690 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0327 22:58:55.371709 3898 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0327 22:58:55.372220 3898 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0327 22:58:55.372236 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0327 22:58:55.372244 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0327 22:58:55.373589 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0327 22:58:55.373605 3898 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0327 22:58:55.373614 3898 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0327 22:58:55.374554 3898 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0327 22:58:55.374572 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0327 22:58:55.374581 3898 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0327 22:58:55.375001 3898 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0327 22:58:55.375017 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0327 22:58:55.375024 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0327 22:58:55.375436 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0327 22:58:55.375451 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0327 22:58:55.375458 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0327 22:58:55.375894 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0327 22:58:55.375911 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0327 22:58:55.375916 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0327 22:58:55.376159 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0327 22:58:55.376174 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0327 22:58:55.376181 3898 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0327 22:58:55.376682 3898 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0327 22:58:55.376699 3898 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0327 22:58:55.376708 3898 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0327 22:58:55.377815 3898 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0327 22:58:55.377835 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0327 22:58:55.377846 3898 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0327 22:58:55.378600 3898 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0327 22:58:55.378623 3898 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0327 22:58:55.378633 3898 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0327 22:58:55.379115 3898 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0327 22:58:55.379136 3898 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0327 22:58:55.379147 3898 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0327 22:58:55.380327 3898 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0327 22:58:55.380689 3898 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-n9kkh of type *factory.egressIPPod in terminal state (e.g. completed) during add event: will remove it I0327 22:58:55.386622 3898 egressfirewall.go:319] Starting EgressFirewall controller I0327 22:58:55.386645 3898 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0327 22:58:55.386665 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0327 22:58:55.386681 3898 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0327 22:58:55.386690 3898 controller.go:133] Adding controller egress-firewall-controller event handlers I0327 22:58:55.386731 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0327 22:58:55.386743 3898 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0327 22:58:55.387033 3898 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0327 22:58:55.387046 3898 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0327 22:58:55.387054 3898 egressqos.go:193] Setting up event handlers for EgressQoS I0327 22:58:55.387133 3898 egressfirewall.go:628] Syncing node "ip-10-0-0-48.ec2.internal" for egress firewall I0327 22:58:55.387146 3898 egressfirewall.go:628] Syncing node "ip-10-0-1-88.ec2.internal" for egress firewall I0327 22:58:55.387152 3898 egressfirewall.go:628] Syncing node "ip-10-0-2-195.ec2.internal" for egress firewall I0327 22:58:55.387266 3898 egressqos.go:245] Starting EgressQoS Controller I0327 22:58:55.387275 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0327 22:58:55.387281 3898 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0327 22:58:55.387286 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0327 22:58:55.387290 3898 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0327 22:58:55.387297 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0327 22:58:55.387303 3898 shared_informer.go:356] "Caches are synced" controller="egressqos" I0327 22:58:55.387309 3898 egressqos.go:259] Repairing EgressQoSes I0327 22:58:55.387315 3898 egressqos.go:399] Starting repairing loop for egressqos I0327 22:58:55.387393 3898 egressqos.go:401] Finished repairing loop for egressqos: 80.608µs I0327 22:58:55.387406 3898 egressservice_zone.go:132] Setting up event handlers for Egress Services I0327 22:58:55.387443 3898 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-48.ec2.internal I0327 22:58:55.387510 3898 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-48.ec2.internal : 68.685µs I0327 22:58:55.387648 3898 egressservice_zone.go:210] Starting Egress Services Controller I0327 22:58:55.387660 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 22:58:55.387668 3898 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 22:58:55.387674 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 22:58:55.387679 3898 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 22:58:55.387686 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 22:58:55.387693 3898 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 22:58:55.387698 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0327 22:58:55.387702 3898 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0327 22:58:55.387706 3898 egressservice_zone.go:228] Repairing Egress Services I0327 22:58:55.387978 3898 master_controller.go:88] Starting Admin Policy Based Route Controller I0327 22:58:55.387988 3898 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 22:58:55.388046 3898 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-48.ec2.internal I0327 22:58:55.388057 3898 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-48.ec2.internal: 10.97µs I0327 22:58:55.388064 3898 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-88.ec2.internal I0327 22:58:55.388071 3898 default_network_controller.go:568] Completing all the Watchers took 311.609ms I0327 22:58:55.388083 3898 default_network_controller.go:572] Starting unidling controllers I0327 22:58:55.388098 3898 unidle.go:46] Registering OVN SB ControllerEvent handler I0327 22:58:55.388073 3898 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-88.ec2.internal: 9.584µs I0327 22:58:55.388106 3898 unidle.go:63] Populating Initial ContollerEvent events I0327 22:58:55.388114 3898 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-195.ec2.internal I0327 22:58:55.388116 3898 unidle.go:79] Setting up event handlers for services I0327 22:58:55.388125 3898 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-195.ec2.internal: 13.464µs I0327 22:58:55.388186 3898 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0327 22:58:55.591357 3898 default_node_network_controller.go:518] Node connection status = connected I0327 22:58:56.091374 3898 default_node_network_controller.go:518] Node connection status = connected I0327 22:58:56.329098 3898 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0327 22:58:56.591218 3898 default_node_network_controller.go:518] Node connection status = connected I0327 22:58:56.607268 3898 bridgeconfig.go:537] Gateway is ready I0327 22:58:56.607290 3898 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0327 22:58:56.617933 3898 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.48. Updating OVN encap IP. I0327 22:58:56.622785 3898 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.48 - it is already configured I0327 22:58:56.622800 3898 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.48/24:{} 172.20.0.1/32:{}]. Updating annotations. I0327 22:58:56.623056 3898 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.48/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-48.ec2.internal","mac-address":"12:34:ec:aa:24:9b","ip-addresses":["10.0.0.48/24"],"ip-address":"10.0.0.48/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:63d81019-9a7c-45e3-a948-15f62537fa6e k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.48/24"}] on node ip-10-0-0-48.ec2.internal I0327 22:58:56.637587 3898 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-48.ec2.internal I0327 22:58:56.637605 3898 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-48.ec2.internal: 20.667µs I0327 22:58:56.637648 3898 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-48.ec2.internal in network "default" I0327 22:58:56.637713 3898 node_tracker.go:169] Node ip-10-0-0-48.ec2.internal switch + router changed, syncing services in network "default" I0327 22:58:56.637719 3898 master.go:532] Adding or Updating local node "ip-10-0-0-48.ec2.internal" for network "default" I0327 22:58:56.637729 3898 services_controller.go:574] Full service sync requested for network=default I0327 22:58:56.637740 3898 egressfirewall.go:628] Syncing node "ip-10-0-0-48.ec2.internal" for egress firewall I0327 22:58:56.638055 3898 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.0.48 W0327 22:58:56.638131 3898 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 22:58:56.638187 3898 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-48.ec2.internal I0327 22:58:56.643236 3898 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-48.ec2.internal. Updating no re-route policies I0327 22:58:56.643325 3898 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-cblx6 I0327 22:58:56.643346 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-7494r I0327 22:58:56.643361 3898 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-cblx6 I0327 22:58:56.643365 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-7494r I0327 22:58:56.643366 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-nrn84 I0327 22:58:56.643366 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-d4gnb I0327 22:58:56.643376 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-b7mkk I0327 22:58:56.643381 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-nrn84 I0327 22:58:56.643386 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-7494r after 0 failed attempt(s) I0327 22:58:56.643386 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-d4gnb I0327 22:58:56.643392 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-b7mkk I0327 22:58:56.643392 3898 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-cblx6 after 0 failed attempt(s) I0327 22:58:56.643392 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-h98kc I0327 22:58:56.643403 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-qgvsg I0327 22:58:56.643400 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-gdslq I0327 22:58:56.643411 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-48.ec2.internal I0327 22:58:56.643411 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-d4gnb after 0 failed attempt(s) I0327 22:58:56.643413 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-qgvsg I0327 22:58:56.643420 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-gdslq I0327 22:58:56.643421 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-48.ec2.internal I0327 22:58:56.643422 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-nrn84 after 0 failed attempt(s) I0327 22:58:56.643434 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-zk7zb I0327 22:58:56.643442 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-48.ec2.internal after 0 failed attempt(s) I0327 22:58:56.643349 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-vqwfp I0327 22:58:56.643497 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-vqwfp I0327 22:58:56.643516 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-vqwfp after 0 failed attempt(s) I0327 22:58:56.643402 3898 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-n7l7j I0327 22:58:56.643528 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-n7l7j I0327 22:58:56.643552 3898 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-n7l7j] creating logical port openshift-multus_network-metrics-daemon-n7l7j for pod on switch ip-10-0-0-48.ec2.internal I0327 22:58:56.643443 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-gdslq after 0 failed attempt(s) I0327 22:58:56.643411 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-h98kc I0327 22:58:56.643644 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-h98kc after 0 failed attempt(s) I0327 22:58:56.643443 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-b7mkk after 0 failed attempt(s) I0327 22:58:56.643448 3898 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-zk7zb I0327 22:58:56.643685 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-zk7zb after 0 failed attempt(s) I0327 22:58:56.643453 3898 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-qgvsg] creating logical port openshift-network-diagnostics_network-check-target-qgvsg for pod on switch ip-10-0-0-48.ec2.internal I0327 22:58:56.643454 3898 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-48.ec2.internal I0327 22:58:56.643710 3898 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-48.ec2.internal I0327 22:58:56.643725 3898 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-48.ec2.internal after 0 failed attempt(s) I0327 22:58:56.644210 3898 pods.go:271] [openshift-multus/network-metrics-daemon-n7l7j] addLogicalPort took 663.055µs, libovsdb time 261.317µs I0327 22:58:56.644223 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-n7l7j after 0 failed attempt(s) I0327 22:58:56.644418 3898 pods.go:271] [openshift-network-diagnostics/network-check-target-qgvsg] addLogicalPort took 980.595µs, libovsdb time 365.344µs I0327 22:58:56.644430 3898 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-qgvsg after 0 failed attempt(s) I0327 22:58:56.652756 3898 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0327 22:58:56.663536 3898 iptables.go:108] Creating table: mangle chain: OUTPUT I0327 22:58:56.666732 3898 iptables.go:108] Creating table: nat chain: OUTPUT I0327 22:58:56.695307 3898 iptables.go:108] Creating table: nat chain: PREROUTING I0327 22:58:56.735125 3898 iptables.go:108] Creating table: filter chain: FORWARD I0327 22:58:56.752728 3898 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0327 22:58:56.752816 3898 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0327 22:58:56.752865 3898 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0327 22:58:56.752933 3898 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0327 22:58:56.752981 3898 gateway_init.go:325] Gateway and management port readiness took 1.665738451s I0327 22:58:56.753015 3898 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0327 22:58:56.753045 3898 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0327 22:58:56.753058 3898 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0327 22:58:56.753067 3898 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 22:58:56.753130 3898 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0327 22:58:56.753233 3898 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 59.066µs I0327 22:58:56.753544 3898 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0327 22:58:56.753581 3898 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 21.169µs I0327 22:58:56.753592 3898 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 79.386µs I0327 22:58:56.753601 3898 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0327 22:58:56.753615 3898 nad_controller.go:173] [node-nad-controller NAD controller]: started I0327 22:58:56.753627 3898 default_node_network_controller.go:898] Starting the default node network controller I0327 22:58:56.753634 3898 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0327 22:58:56.753715 3898 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 16.765µs I0327 22:58:56.753831 3898 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0327 22:58:56.753841 3898 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0327 22:58:56.753864 3898 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000ae8660 0xc0002dc9a0 0xc0002dca10 0xc0002dcaf0 0xc0002dcb60 0xc0002dcbd0 0xc0002dcc40 0xc000acf090 0xc0002dcd20 0xc0002dcd90 0xc00086a780 0xc0064f5180 0xc0002dce00 map[0x2f67040:0xc000cb3c20 0x2f67580:0xc000cb3da0 0x2f68000:0xc002294480 0x2f682a0:0xc0026ab8c0 0x2f687e0:0xc000cb3c80 0x2f68a80:0xc000cb3ce0 0x2f68d20:0xc000cb3d40 0x2f69500:0xc000cb3e00 0x2f697a0:0xc0066020c0 0x2f93d40:0xc000e90660 0x2f940e0:0xc000e906c0 0x2f94bc0:0xc000ae5e00 0x2f96520:0xc001b20000 0x2f97000:0xc000e907e0 0x2f97e80:0xc001b20060] 0xc0006ab1f0 0} I0327 22:58:56.753917 3898 gateway.go:151] Starting gateway service sync I0327 22:58:56.754397 3898 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0327 22:58:56.756644 3898 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0327 22:58:56.766603 3898 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0327 22:58:56.767874 3898 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0327 22:58:56.769043 3898 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0327 22:58:56.808582 3898 gateway.go:168] Gateway service sync done. Time taken: 54.651044ms I0327 22:58:56.809588 3898 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30195 I0327 22:58:56.809835 3898 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000ae8660 0xc0002dc9a0 0xc0002dca10 0xc0002dcaf0 0xc0002dcb60 0xc0002dcbd0 0xc0002dcc40 0xc000acf090 0xc0002dcd20 0xc0002dcd90 0xc00086a780 0xc0064f5180 0xc0002dce00 map[0x2f67040:0xc000cb3c20 0x2f67580:0xc000cb3da0 0x2f68000:0xc002294480 0x2f682a0:0xc0026ab8c0 0x2f687e0:0xc000cb3c80 0x2f68a80:0xc000cb3ce0 0x2f68d20:0xc000cb3d40 0x2f69500:0xc000cb3e00 0x2f697a0:0xc0066020c0 0x2f93d40:0xc000e90660 0x2f940e0:0xc000e906c0 0x2f94bc0:0xc000ae5e00 0x2f96520:0xc001b20000 0x2f97000:0xc000e907e0 0x2f97e80:0xc001b20060] 0xc0006ab1f0 0} I0327 22:58:56.810608 3898 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000ae8660 0xc0002dc9a0 0xc0002dca10 0xc0002dcaf0 0xc0002dcb60 0xc0002dcbd0 0xc0002dcc40 0xc000acf090 0xc0002dcd20 0xc0002dcd90 0xc00086a780 0xc0064f5180 0xc0002dce00 map[0x2f67040:0xc000cb3c20 0x2f67580:0xc000cb3da0 0x2f68000:0xc002294480 0x2f682a0:0xc0026ab8c0 0x2f687e0:0xc000cb3c80 0x2f68a80:0xc000cb3ce0 0x2f68d20:0xc000cb3d40 0x2f69500:0xc000cb3e00 0x2f697a0:0xc0066020c0 0x2f93d40:0xc000e90660 0x2f940e0:0xc000e906c0 0x2f94bc0:0xc000ae5e00 0x2f96520:0xc001b20000 0x2f97000:0xc000e907e0 0x2f97e80:0xc001b20060] 0xc0006ab1f0 0} I0327 22:58:56.810658 3898 gateway.go:519] Reconciling gateway with updates I0327 22:58:56.816932 3898 gateway.go:326] Spawning Conntrack Rule Check Thread I0327 22:58:56.817024 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 22:58:56.817040 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 22:58:56.817046 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 22:58:56.817058 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 22:58:56.817062 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 22:58:56.817058 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 22:58:56.817067 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 22:58:56.817076 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 22:58:56.817095 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 22:58:56.817100 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 22:58:56.817103 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 22:58:56.817121 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 22:58:56.817139 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 22:58:56.817144 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 22:58:56.817148 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 22:58:56.817160 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 22:58:56.817161 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 22:58:56.817168 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 22:58:56.817082 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 22:58:56.817177 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 22:58:56.817180 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 22:58:56.817181 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 22:58:56.817186 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 22:58:56.817190 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 22:58:56.817192 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 22:58:56.817190 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 22:58:56.817196 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 22:58:56.817197 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 22:58:56.817187 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 22:58:56.817210 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 22:58:56.817213 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817219 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 22:58:56.817222 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 22:58:56.817233 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 22:58:56.817232 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 22:58:56.817233 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 22:58:56.817238 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 22:58:56.817246 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817252 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 22:58:56.817253 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 22:58:56.817256 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 22:58:56.817254 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 22:58:56.817262 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 22:58:56.817265 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 22:58:56.817268 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 22:58:56.817267 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 22:58:56.817274 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 22:58:56.817215 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 22:58:56.817280 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 22:58:56.817277 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 22:58:56.817105 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 22:58:56.817293 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 22:58:56.817125 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 22:58:56.817298 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 22:58:56.817110 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 22:58:56.817248 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 22:58:56.817303 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 22:58:56.817308 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 22:58:56.817313 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 22:58:56.817317 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 22:58:56.817320 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 22:58:56.817332 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 22:58:56.817340 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 22:58:56.817344 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 22:58:56.817352 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 22:58:56.817300 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 22:58:56.817148 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 22:58:56.817388 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 22:58:56.817395 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 22:58:56.817396 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 22:58:56.817403 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 22:58:56.817407 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 22:58:56.817411 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 22:58:56.817415 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 22:58:56.817423 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 22:58:56.817427 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 22:58:56.817350 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817126 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 22:58:56.817128 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817034 3898 default_node_network_controller.go:943] Gateway and management port readiness took 63.18665ms I0327 22:58:56.817440 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817443 3898 port_claim.go:71] Svc openshift-ingress/router-default: port 31577 is already open, no action needed I0327 22:58:56.817443 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817449 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 22:58:56.817283 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 22:58:56.817454 3898 port_claim.go:71] Svc openshift-ingress/router-default: port 31084 is already open, no action needed I0327 22:58:56.817167 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 22:58:56.817126 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 22:58:56.817500 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 22:58:56.817508 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 22:58:56.817142 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 22:58:56.817524 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 22:58:56.817528 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 22:58:56.817172 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 22:58:56.817167 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 22:58:56.817171 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 22:58:56.817562 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 22:58:56.817197 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 22:58:56.817221 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 22:58:56.817605 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 22:58:56.817244 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 22:58:56.817619 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 22:58:56.817256 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817270 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 22:58:56.817276 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817292 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 22:58:56.817292 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 22:58:56.817307 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 22:58:56.817384 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 22:58:56.817789 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 22:58:56.817383 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 22:58:56.817855 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 22:58:56.817408 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 22:58:56.817907 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 22:58:56.817459 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817127 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 22:58:56.817148 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 22:58:56.817988 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 22:58:56.817997 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 22:58:56.817181 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 22:58:56.818006 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 22:58:56.818562 3898 node_ip_handler_linux.go:160] Node IP manager is running W0327 22:58:56.855857 3898 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 22:58:56.856131 3898 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 22:58:56.856395 3898 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0327 22:58:56.867108 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 22:58:56.867124 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 22:58:56.867147 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 22:58:56.867159 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 22:58:56.867170 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 22:58:56.906699 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 22:58:56.906731 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 22:58:56.906741 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 22:58:56.906748 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 22:58:56.906754 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 22:58:56.906762 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 22:58:56.906769 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 22:58:56.906776 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 22:58:56.949641 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 22:58:58.724102 3898 default_node_network_controller.go:1364] Starting node controller node sync W0327 22:58:58.724121 3898 egressip_healthcheck.go:76] Health checking using insecure connection I0327 22:58:58.724170 3898 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0327 22:58:58.736016 3898 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.891637ms I0327 22:58:58.736160 3898 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-88.ec2.internal", IP: 10.0.1.88 to PMTUD blocking rules I0327 22:58:58.736174 3898 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-88.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 22:58:58.736199 3898 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-195.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 22:58:58.736215 3898 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-195.ec2.internal", IP: 10.0.2.195 to PMTUD blocking rules I0327 22:58:58.748176 3898 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0327 22:58:58.748459 3898 egressservice_node.go:93] Setting up event handlers for Egress Services I0327 22:58:58.748994 3898 egressservice_node.go:183] Starting Egress Services Controller I0327 22:58:58.749016 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 22:58:58.749028 3898 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 22:58:58.749036 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 22:58:58.749041 3898 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 22:58:58.749048 3898 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 22:58:58.749062 3898 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 22:58:58.749068 3898 egressservice_node.go:197] Repairing Egress Services I0327 22:58:58.759233 3898 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0327 22:58:58.759245 3898 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 22:58:58.759436 3898 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0327 22:58:58.759453 3898 link_network_manager.go:117] Link manager is running I0327 22:58:58.759479 3898 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0327 22:58:58.759561 3898 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0327 22:58:58.759944 3898 vrf_manager.go:128] VRF manager is running I0327 22:58:58.760064 3898 node_controller_manager.go:437] Removing flows to drop GARP I0327 22:58:58.760074 3898 gateway.go:519] Reconciling gateway with updates I0327 22:58:58.780617 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 22:58:58.780636 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 22:58:58.780636 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 22:58:58.780617 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 22:58:58.780650 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 22:58:58.780655 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 22:58:58.780658 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 22:58:58.780656 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 22:58:58.780668 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 22:58:58.780674 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 22:58:58.780683 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 22:58:58.780683 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 22:58:58.780696 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 22:58:58.780698 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 22:58:58.780712 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 22:58:58.780682 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 22:58:58.780728 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 22:58:58.780740 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 22:58:58.780741 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 22:58:58.780745 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 22:58:58.780762 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 22:58:58.780766 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 22:58:58.780774 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 22:58:58.780778 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 22:58:58.780781 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 22:58:58.780785 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 22:58:58.780787 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 22:58:58.780790 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 22:58:58.780796 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 22:58:58.780802 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 22:58:58.780805 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 22:58:58.780814 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 22:58:58.780819 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 22:58:58.780825 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 22:58:58.780823 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 22:58:58.780836 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 22:58:58.780838 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 22:58:58.780780 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 22:58:58.780851 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 22:58:58.780856 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 22:58:58.780644 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 22:58:58.780864 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 22:58:58.780866 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 22:58:58.780742 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 22:58:58.780868 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 22:58:58.780881 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 22:58:58.780886 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 22:58:58.780888 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 22:58:58.780894 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 22:58:58.780866 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 22:58:58.780902 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 22:58:58.780904 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 22:58:58.780914 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 22:58:58.780939 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 22:58:58.780920 3898 port_claim.go:71] Svc openshift-ingress/router-default: port 31577 is already open, no action needed I0327 22:58:58.780952 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 22:58:58.780956 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 22:58:58.780778 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 22:58:58.780966 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 22:58:58.780892 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 22:58:58.780957 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 22:58:58.780715 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 22:58:58.781006 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 22:58:58.780944 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 22:58:58.781022 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 22:58:58.780826 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 22:58:58.781032 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 22:58:58.781037 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 22:58:58.780671 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 22:58:58.781048 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 22:58:58.781054 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 22:58:58.780765 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 22:58:58.781062 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 22:58:58.781066 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 22:58:58.781068 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 22:58:58.781072 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 22:58:58.781079 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 22:58:58.781091 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 22:58:58.781133 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 22:58:58.780803 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 22:58:58.781162 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 22:58:58.780789 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 22:58:58.781035 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 22:58:58.780813 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 22:58:58.781196 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 22:58:58.781218 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 22:58:58.781238 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 22:58:58.781240 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 22:58:58.781251 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 22:58:58.780751 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 22:58:58.781296 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 22:58:58.781295 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 22:58:58.781327 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 22:58:58.780756 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 22:58:58.781345 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 22:58:58.781382 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 22:58:58.780803 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 22:58:58.781398 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 22:58:58.781432 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 22:58:58.780843 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 22:58:58.781486 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 22:58:58.780844 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 22:58:58.780855 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 22:58:58.780859 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 22:58:58.781564 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 22:58:58.780877 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 22:58:58.781603 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 22:58:58.780875 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 22:58:58.781616 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 22:58:58.780888 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 22:58:58.781655 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 22:58:58.781663 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 22:58:58.780895 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 22:58:58.780948 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 22:58:58.780972 3898 port_claim.go:71] Svc openshift-ingress/router-default: port 31084 is already open, no action needed I0327 22:58:58.780976 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 22:58:58.781026 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 22:58:58.781045 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 22:58:58.781059 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 22:58:58.781056 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 22:58:58.781784 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 22:58:58.781283 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 22:58:58.780741 3898 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 22:58:58.781824 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 22:58:58.780727 3898 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 22:58:58.802377 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 22:58:58.821708 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 22:58:58.821796 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 22:58:58.862563 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 22:58:58.862633 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 22:58:58.862655 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 22:58:58.862672 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 22:58:58.862690 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 22:58:58.862699 3898 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 22:59:00.615582 3898 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-48.ec2.internal I0327 22:59:00.615611 3898 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-48.ec2.internal: 35.27µs I0327 22:59:00.646232 3898 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-57996bf879-5hf57] creating logical port openshift-image-registry_image-registry-57996bf879-5hf57 for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:00.646353 3898 kube.go:256] Updating pod openshift-image-registry/image-registry-57996bf879-5hf57 I0327 22:59:00.664278 3898 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-vwcb2] creating logical port openshift-ingress-canary_ingress-canary-vwcb2 for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:00.664413 3898 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-vwcb2 I0327 22:59:00.666705 3898 pod.go:62] [openshift-image-registry/image-registry-57996bf879-5hf57] pod update took 20.37838ms I0327 22:59:00.666725 3898 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-57996bf879-5hf57] addLogicalPort annotation time took 20.402289ms I0327 22:59:00.667790 3898 pods.go:271] [openshift-image-registry/image-registry-57996bf879-5hf57] addLogicalPort took 21.571186ms, libovsdb time 834.529µs I0327 22:59:00.669914 3898 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-kzdgm] creating logical port openshift-dns_dns-default-kzdgm for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:00.670009 3898 kube.go:256] Updating pod openshift-dns/dns-default-kzdgm I0327 22:59:00.681979 3898 pod.go:62] [openshift-ingress-canary/ingress-canary-vwcb2] pod update took 17.58763ms I0327 22:59:00.682001 3898 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-vwcb2] addLogicalPort annotation time took 17.622951ms I0327 22:59:00.682967 3898 pods.go:271] [openshift-ingress-canary/ingress-canary-vwcb2] addLogicalPort took 18.701476ms, libovsdb time 733.489µs I0327 22:59:00.685761 3898 pod.go:62] [openshift-dns/dns-default-kzdgm] pod update took 15.78898ms I0327 22:59:00.685777 3898 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-kzdgm] addLogicalPort annotation time took 15.806182ms I0327 22:59:00.686655 3898 pods.go:271] [openshift-dns/dns-default-kzdgm] addLogicalPort took 16.782562ms, libovsdb time 658.298µs I0327 22:59:01.024329 3898 cni.go:403] [openshift-ingress-canary/ingress-canary-vwcb2 5f30f75bf529a2f690dc47fe5411648b7701d7e3e0c86afc2e21b5a97d4aa49b network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-vwcb2 5f30f75bf529a2f690dc47fe5411648b7701d7e3e0c86afc2e21b5a97d4aa49b network default NAD default] I0327 22:59:01.040185 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-vwcb2, hostIfaceName: 5f30f75bf529a2f, network: default, NAD default, SandboxID: "5f30f75bf529a2f690dc47fe5411648b7701d7e3e0c86afc2e21b5a97d4aa49b", PCI device ID: , UID: "32a7dc02-e226-41c2-9292-8ae8529f5188", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0327 22:59:01.089578 3898 cni.go:424] [openshift-ingress-canary/ingress-canary-vwcb2 5f30f75bf529a2f690dc47fe5411648b7701d7e3e0c86afc2e21b5a97d4aa49b network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-vwcb2 5f30f75bf529a2f690dc47fe5411648b7701d7e3e0c86afc2e21b5a97d4aa49b network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:0c:b6:76:8a:2c\",\"name\":\"5f30f75bf529a2f\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a875c977-09d6-4985-9766-e69374b33e34\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 22:59:05.408128 3898 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:05.408745 3898 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:05.408773 3898 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 22:59:05.408782 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0327 22:59:05.408802 3898 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 22:59:05.408810 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0327 22:59:06.433280 3898 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:06.433304 3898 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:06.433324 3898 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:06.433336 3898 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390hclv9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:06.834383 3898 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 22:59:06.844604 3898 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 22:59:21.883330 3898 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-nfxk9] creating logical port openshift-service-ca-operator_service-ca-operator-7b89bc695f-nfxk9 for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:21.883422 3898 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-7b89bc695f-nfxk9 I0327 22:59:21.883566 3898 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-8wvst] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-6ff64bb8b6-8wvst for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:21.883661 3898 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-8wvst I0327 22:59:21.910565 3898 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-8wvst] pod update took 26.920296ms I0327 22:59:21.910595 3898 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-8wvst] addLogicalPort annotation time took 26.953193ms I0327 22:59:21.910678 3898 pod.go:62] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-nfxk9] pod update took 27.281855ms I0327 22:59:21.910695 3898 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-nfxk9] addLogicalPort annotation time took 27.302443ms I0327 22:59:21.911778 3898 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-8wvst] addLogicalPort took 28.222043ms, libovsdb time 840.604µs I0327 22:59:21.912084 3898 pods.go:271] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-nfxk9] addLogicalPort took 28.775939ms, libovsdb time 1.142509ms I0327 22:59:21.974201 3898 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-5577457d66-hhjgn] creating logical port openshift-insights_insights-operator-5577457d66-hhjgn for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:21.974286 3898 kube.go:256] Updating pod openshift-insights/insights-operator-5577457d66-hhjgn I0327 22:59:21.991008 3898 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-wlwn6] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67b99c68bb-wlwn6 for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:21.991091 3898 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-wlwn6 I0327 22:59:21.991356 3898 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-j9t46] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-555cc59f77-j9t46 for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:21.991439 3898 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-wdwj4] creating logical port openshift-console-operator_console-operator-8777bd9bb-wdwj4 for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:21.991523 3898 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-j9t46 I0327 22:59:21.991537 3898 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-wdwj4 I0327 22:59:22.010088 3898 pod.go:62] [openshift-insights/insights-operator-5577457d66-hhjgn] pod update took 35.820793ms I0327 22:59:22.010114 3898 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-5577457d66-hhjgn] addLogicalPort annotation time took 35.852034ms I0327 22:59:22.011209 3898 pods.go:271] [openshift-insights/insights-operator-5577457d66-hhjgn] addLogicalPort took 37.023988ms, libovsdb time 774.501µs I0327 22:59:22.016059 3898 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-wlwn6] pod update took 24.990507ms I0327 22:59:22.016079 3898 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-wlwn6] addLogicalPort annotation time took 25.01472ms I0327 22:59:22.017286 3898 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-wlwn6] addLogicalPort took 26.289156ms, libovsdb time 919.39µs I0327 22:59:22.018819 3898 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-j9t46] pod update took 27.379622ms I0327 22:59:22.018836 3898 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-j9t46] addLogicalPort annotation time took 27.400775ms I0327 22:59:22.019868 3898 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-j9t46] addLogicalPort took 28.520367ms, libovsdb time 819.768µs I0327 22:59:22.024274 3898 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-wdwj4] pod update took 32.764775ms I0327 22:59:22.024291 3898 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-wdwj4] addLogicalPort annotation time took 32.783479ms I0327 22:59:22.025313 3898 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-wdwj4] addLogicalPort took 33.885341ms, libovsdb time 774.456µs I0327 22:59:22.081923 3898 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-qhcpn] creating logical port openshift-monitoring_cluster-monitoring-operator-6dd897695f-qhcpn for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:22.081994 3898 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-6dd897695f-qhcpn I0327 22:59:22.098844 3898 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-qhcpn] pod update took 16.869241ms I0327 22:59:22.098864 3898 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-qhcpn] addLogicalPort annotation time took 16.892299ms I0327 22:59:22.099740 3898 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-qhcpn] addLogicalPort took 17.831908ms, libovsdb time 639.694µs I0327 22:59:22.101731 3898 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-q97k5] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-q97k5 for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:22.101821 3898 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-q97k5 I0327 22:59:22.121008 3898 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-q97k5] pod update took 19.203961ms I0327 22:59:22.121031 3898 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-q97k5] addLogicalPort annotation time took 19.229143ms I0327 22:59:22.122011 3898 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-q97k5] addLogicalPort took 20.289763ms, libovsdb time 712.639µs I0327 22:59:22.271222 3898 cni.go:403] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-nfxk9 5f0c667109ac677fa1d312ba3af67e90d7d2d01a3e0b191572a040af620e22e4 network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-nfxk9 5f0c667109ac677fa1d312ba3af67e90d7d2d01a3e0b191572a040af620e22e4 network default NAD default] I0327 22:59:22.279617 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-7b89bc695f-nfxk9, hostIfaceName: 5f0c667109ac677, network: default, NAD default, SandboxID: "5f0c667109ac677fa1d312ba3af67e90d7d2d01a3e0b191572a040af620e22e4", PCI device ID: , UID: "e1ac28ef-9008-4620-8867-6b9bfe9c953e", MAC: 0a:58:0a:82:00:08, IPs: [10.130.0.8/23] I0327 22:59:22.280248 3898 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-8wvst 3aef1b86a58c03163e83f9574db8eec63c438e72eba75c0603af0b61c615150c network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-8wvst 3aef1b86a58c03163e83f9574db8eec63c438e72eba75c0603af0b61c615150c network default NAD default] I0327 22:59:22.289742 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-6ff64bb8b6-8wvst, hostIfaceName: 3aef1b86a58c031, network: default, NAD default, SandboxID: "3aef1b86a58c03163e83f9574db8eec63c438e72eba75c0603af0b61c615150c", PCI device ID: , UID: "e1ec7ba7-25d0-4f80-a27e-1ed6b03f5714", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0327 22:59:22.321676 3898 cni.go:424] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-nfxk9 5f0c667109ac677fa1d312ba3af67e90d7d2d01a3e0b191572a040af620e22e4 network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-nfxk9 5f0c667109ac677fa1d312ba3af67e90d7d2d01a3e0b191572a040af620e22e4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:7b:ca:c1:e9:25\",\"name\":\"5f0c667109ac677\"},{\"mac\":\"0a:58:0a:82:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/aa055fe3-5c81-4084-9466-8fbe919e706e\"}],\"ips\":[{\"address\":\"10.130.0.8/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 22:59:22.336697 3898 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-8wvst 3aef1b86a58c03163e83f9574db8eec63c438e72eba75c0603af0b61c615150c network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-8wvst 3aef1b86a58c03163e83f9574db8eec63c438e72eba75c0603af0b61c615150c network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:99:a5:68:17:1d\",\"name\":\"3aef1b86a58c031\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/301505aa-af33-41f7-b39a-c74852a84158\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 22:59:22.371069 3898 cni.go:403] [openshift-insights/insights-operator-5577457d66-hhjgn de3519fee295ae3ab10df5b1623664cfa5eb3c2daea95f274e64b86f0837f469 network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-5577457d66-hhjgn de3519fee295ae3ab10df5b1623664cfa5eb3c2daea95f274e64b86f0837f469 network default NAD default] I0327 22:59:22.380451 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-5577457d66-hhjgn, hostIfaceName: de3519fee295ae3, network: default, NAD default, SandboxID: "de3519fee295ae3ab10df5b1623664cfa5eb3c2daea95f274e64b86f0837f469", PCI device ID: , UID: "fc0983fe-a349-472e-b25d-a873ee59c802", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0327 22:59:22.400593 3898 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-j9t46 e2dcba31486b757a597e19f20f1be15073c0822cd50727694d9fee39a0ce0d97 network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-j9t46 e2dcba31486b757a597e19f20f1be15073c0822cd50727694d9fee39a0ce0d97 network default NAD default] I0327 22:59:22.408902 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-555cc59f77-j9t46, hostIfaceName: e2dcba31486b757, network: default, NAD default, SandboxID: "e2dcba31486b757a597e19f20f1be15073c0822cd50727694d9fee39a0ce0d97", PCI device ID: , UID: "c18bb043-1977-4c91-b7f3-f2e387de4add", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0327 22:59:22.422649 3898 cni.go:424] [openshift-insights/insights-operator-5577457d66-hhjgn de3519fee295ae3ab10df5b1623664cfa5eb3c2daea95f274e64b86f0837f469 network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-5577457d66-hhjgn de3519fee295ae3ab10df5b1623664cfa5eb3c2daea95f274e64b86f0837f469 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:9a:bd:dd:94:6e\",\"name\":\"de3519fee295ae3\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6f419418-7b80-45c1-a5d4-9cfa500ebbb4\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 22:59:22.423510 3898 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-wdwj4 93268f43669da11d6c72e69bc1100f8f45422ca0b562cf8a00c11ec1fb126299 network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-wdwj4 93268f43669da11d6c72e69bc1100f8f45422ca0b562cf8a00c11ec1fb126299 network default NAD default] I0327 22:59:22.434204 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-wdwj4, hostIfaceName: 93268f43669da11, network: default, NAD default, SandboxID: "93268f43669da11d6c72e69bc1100f8f45422ca0b562cf8a00c11ec1fb126299", PCI device ID: , UID: "48940c0f-07b9-4d45-95d3-ffd769ffe576", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0327 22:59:22.452593 3898 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-j9t46 e2dcba31486b757a597e19f20f1be15073c0822cd50727694d9fee39a0ce0d97 network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-j9t46 e2dcba31486b757a597e19f20f1be15073c0822cd50727694d9fee39a0ce0d97 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:cd:24:1f:fd:47\",\"name\":\"e2dcba31486b757\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7df2114e-2223-42a9-a7eb-26c3bf3b4783\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 22:59:22.473927 3898 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-wdwj4 93268f43669da11d6c72e69bc1100f8f45422ca0b562cf8a00c11ec1fb126299 network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-wdwj4 93268f43669da11d6c72e69bc1100f8f45422ca0b562cf8a00c11ec1fb126299 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:6e:c3:55:26:9f\",\"name\":\"93268f43669da11\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/117355de-1aee-4a3a-87c5-86ffdd692cad\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 22:59:28.026631 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 22:59:28.391794 3898 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0327 22:59:28.392744 3898 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 928.231µs I0327 22:59:28.427518 3898 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 22:59:29.464028 3898 namespace.go:98] [openshift-service-ca] adding namespace I0327 22:59:29.465044 3898 namespace.go:102] [openshift-service-ca] adding namespace took 991.767µs I0327 22:59:29.479500 3898 namespace.go:142] [openshift-service-ca] updating namespace I0327 22:59:31.769900 3898 cni.go:403] [openshift-network-diagnostics/network-check-target-qgvsg 24ca8d6240a01305c7e211b04db93c445bd21fb3357c88ab30972a56ec04ed29 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-qgvsg 24ca8d6240a01305c7e211b04db93c445bd21fb3357c88ab30972a56ec04ed29 network default NAD default] I0327 22:59:31.779954 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-qgvsg, hostIfaceName: 24ca8d6240a0130, network: default, NAD default, SandboxID: "24ca8d6240a01305c7e211b04db93c445bd21fb3357c88ab30972a56ec04ed29", PCI device ID: , UID: "b7721865-ba77-404a-984e-bf51d064250a", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0327 22:59:31.820238 3898 cni.go:424] [openshift-network-diagnostics/network-check-target-qgvsg 24ca8d6240a01305c7e211b04db93c445bd21fb3357c88ab30972a56ec04ed29 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-qgvsg 24ca8d6240a01305c7e211b04db93c445bd21fb3357c88ab30972a56ec04ed29 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:b3:11:11:be:c8\",\"name\":\"24ca8d6240a0130\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0a49cddf-b4cf-4319-97c6-0d5e72174d19\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 22:59:37.596992 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 22:59:37.972732 3898 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-wlwn6 6ecbc0e5ebaaa827c3e4366b4cded5250a3e701ab6b6b234ba8e74a46925fbd9 network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-wlwn6 6ecbc0e5ebaaa827c3e4366b4cded5250a3e701ab6b6b234ba8e74a46925fbd9 network default NAD default] I0327 22:59:37.981304 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67b99c68bb-wlwn6, hostIfaceName: 6ecbc0e5ebaaa82, network: default, NAD default, SandboxID: "6ecbc0e5ebaaa827c3e4366b4cded5250a3e701ab6b6b234ba8e74a46925fbd9", PCI device ID: , UID: "8ccf2d99-b0fb-4373-a727-b1f75c45b0cf", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0327 22:59:38.023669 3898 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-wlwn6 6ecbc0e5ebaaa827c3e4366b4cded5250a3e701ab6b6b234ba8e74a46925fbd9 network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-wlwn6 6ecbc0e5ebaaa827c3e4366b4cded5250a3e701ab6b6b234ba8e74a46925fbd9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:1a:d1:5f:62:50\",\"name\":\"6ecbc0e5ebaaa82\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7e9c945c-53a2-4fd4-a50d-dff37f500677\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 22:59:53.963909 3898 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-qhcpn 50bc60ed9e66d4b25cee16f6d09c009bb6d594823620b2574524230e5c8a80ca network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-qhcpn 50bc60ed9e66d4b25cee16f6d09c009bb6d594823620b2574524230e5c8a80ca network default NAD default] I0327 22:59:53.973266 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-6dd897695f-qhcpn, hostIfaceName: 50bc60ed9e66d4b, network: default, NAD default, SandboxID: "50bc60ed9e66d4b25cee16f6d09c009bb6d594823620b2574524230e5c8a80ca", PCI device ID: , UID: "c07fcdd3-b53e-4fd9-8b46-bec6a4983a78", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0327 22:59:54.017118 3898 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-qhcpn 50bc60ed9e66d4b25cee16f6d09c009bb6d594823620b2574524230e5c8a80ca network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-qhcpn 50bc60ed9e66d4b25cee16f6d09c009bb6d594823620b2574524230e5c8a80ca network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:54:4a:0b:5c:ec\",\"name\":\"50bc60ed9e66d4b\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/24838d0c-bf6b-4ae6-8611-9d753bfa5f89\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 22:59:54.271236 3898 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-q97k5 1da8f414225996bbbf1f9f923d2e5554d1075fc26a1df48bc190a2a825f6f120 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-q97k5 1da8f414225996bbbf1f9f923d2e5554d1075fc26a1df48bc190a2a825f6f120 network default NAD default] I0327 22:59:54.279899 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-q97k5, hostIfaceName: 1da8f414225996b, network: default, NAD default, SandboxID: "1da8f414225996bbbf1f9f923d2e5554d1075fc26a1df48bc190a2a825f6f120", PCI device ID: , UID: "3dcf31c6-285a-441d-8ebe-0c0102145c82", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0327 22:59:54.319758 3898 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-q97k5 1da8f414225996bbbf1f9f923d2e5554d1075fc26a1df48bc190a2a825f6f120 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-q97k5 1da8f414225996bbbf1f9f923d2e5554d1075fc26a1df48bc190a2a825f6f120 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:2c:2b:da:3e:f7\",\"name\":\"1da8f414225996b\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d0a1ede9-ffa2-42d9-873c-f86438755494\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 22:59:55.096737 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 22:59:55.096762 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 22:59:55.096796 3898 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 22:59:55.102750 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 22:59:55.102765 3898 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 22:59:55.102772 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 22:59:55.103647 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 22:59:55.266110 3898 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-5jdx9] creating logical port openshift-insights_insights-runtime-extractor-5jdx9 for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:55.266215 3898 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-5jdx9 I0327 22:59:55.287991 3898 pod.go:62] [openshift-insights/insights-runtime-extractor-5jdx9] pod update took 21.81631ms I0327 22:59:55.288014 3898 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-5jdx9] addLogicalPort annotation time took 21.844276ms I0327 22:59:55.289361 3898 pods.go:271] [openshift-insights/insights-runtime-extractor-5jdx9] addLogicalPort took 23.264204ms, libovsdb time 999.731µs I0327 22:59:56.069162 3898 cni.go:403] [openshift-insights/insights-runtime-extractor-5jdx9 08c9ba8a991c691118f8ae5af4650e0abee38c94673978cd27ad37d04a7ca6f2 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-5jdx9 08c9ba8a991c691118f8ae5af4650e0abee38c94673978cd27ad37d04a7ca6f2 network default NAD default] I0327 22:59:56.077754 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-5jdx9, hostIfaceName: 08c9ba8a991c691, network: default, NAD default, SandboxID: "08c9ba8a991c691118f8ae5af4650e0abee38c94673978cd27ad37d04a7ca6f2", PCI device ID: , UID: "960407d4-f4f2-489a-9c12-1d6d9173ac96", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0327 22:59:56.121436 3898 cni.go:424] [openshift-insights/insights-runtime-extractor-5jdx9 08c9ba8a991c691118f8ae5af4650e0abee38c94673978cd27ad37d04a7ca6f2 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-5jdx9 08c9ba8a991c691118f8ae5af4650e0abee38c94673978cd27ad37d04a7ca6f2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:15:4b:9a:6e:34\",\"name\":\"08c9ba8a991c691\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb41fbf5-ab4c-4cd9-bd5b-4943785b1330\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 22:59:56.471061 3898 obj_retry.go:464] Detected object openshift-image-registry/image-registry-656cf7696d-955wr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:56.471628 3898 obj_retry.go:464] Detected object openshift-image-registry/image-registry-656cf7696d-955wr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:56.490370 3898 obj_retry.go:458] Detected object openshift-image-registry/image-registry-656cf7696d-955wr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:56.490391 3898 obj_retry.go:458] Detected object openshift-image-registry/image-registry-656cf7696d-955wr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:56.493829 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-656cf7696d-955wr I0327 22:59:56.493843 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-656cf7696d-955wr I0327 22:59:56.681215 3898 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-qcwkv] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-qcwkv for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:56.681328 3898 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-qcwkv I0327 22:59:56.695527 3898 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-qcwkv] pod update took 14.23575ms I0327 22:59:56.695552 3898 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-qcwkv] addLogicalPort annotation time took 14.265974ms I0327 22:59:56.696857 3898 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-qcwkv] addLogicalPort took 15.669589ms, libovsdb time 983.57µs I0327 22:59:57.039082 3898 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-qcwkv c6e2f10e02c918531a31c9cc85a56e6a776a9d57df41c1110008dfe564086ee0 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-qcwkv c6e2f10e02c918531a31c9cc85a56e6a776a9d57df41c1110008dfe564086ee0 network default NAD default] I0327 22:59:57.047493 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-qcwkv, hostIfaceName: c6e2f10e02c9185, network: default, NAD default, SandboxID: "c6e2f10e02c918531a31c9cc85a56e6a776a9d57df41c1110008dfe564086ee0", PCI device ID: , UID: "1ec8c9f6-15fc-4085-ba42-b2397673b700", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0327 22:59:57.088864 3898 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-qcwkv c6e2f10e02c918531a31c9cc85a56e6a776a9d57df41c1110008dfe564086ee0 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-qcwkv c6e2f10e02c918531a31c9cc85a56e6a776a9d57df41c1110008dfe564086ee0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:95:a7:83:d4:d9\",\"name\":\"c6e2f10e02c9185\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/22dd24e9-aa8d-439b-8b0f-8b9158883ce5\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 22:59:57.148604 3898 obj_retry.go:464] Detected object openshift-image-registry/image-registry-57996bf879-5hf57 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:57.148638 3898 pods.go:176] Deleting pod: openshift-image-registry/image-registry-57996bf879-5hf57 I0327 22:59:57.149871 3898 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-57996bf879-5hf57, ips: 10.130.0.5 I0327 22:59:57.149900 3898 obj_retry.go:464] Detected object openshift-image-registry/image-registry-57996bf879-5hf57 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 22:59:57.156894 3898 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-656f8959d9-s9w6v] creating logical port openshift-image-registry_image-registry-656f8959d9-s9w6v for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:57.156994 3898 kube.go:256] Updating pod openshift-image-registry/image-registry-656f8959d9-s9w6v I0327 22:59:57.162440 3898 obj_retry.go:458] Detected object openshift-image-registry/image-registry-57996bf879-5hf57 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:57.162459 3898 obj_retry.go:458] Detected object openshift-image-registry/image-registry-57996bf879-5hf57 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 22:59:57.172840 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-57996bf879-5hf57 I0327 22:59:57.172861 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-57996bf879-5hf57 I0327 22:59:57.173900 3898 pod.go:62] [openshift-image-registry/image-registry-656f8959d9-s9w6v] pod update took 16.941885ms I0327 22:59:57.173921 3898 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-656f8959d9-s9w6v] addLogicalPort annotation time took 16.965364ms I0327 22:59:57.175165 3898 pods.go:271] [openshift-image-registry/image-registry-656f8959d9-s9w6v] addLogicalPort took 18.284339ms, libovsdb time 961.941µs W0327 22:59:57.478091 3898 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 22:59:57.583504 3898 cni.go:403] [openshift-image-registry/image-registry-656f8959d9-s9w6v e712050acf8f2d0cea9874a06285c2a59e26d091a7e196e05c01d6e8e5560204 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-656f8959d9-s9w6v e712050acf8f2d0cea9874a06285c2a59e26d091a7e196e05c01d6e8e5560204 network default NAD default] I0327 22:59:57.591783 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-656f8959d9-s9w6v, hostIfaceName: e712050acf8f2d0, network: default, NAD default, SandboxID: "e712050acf8f2d0cea9874a06285c2a59e26d091a7e196e05c01d6e8e5560204", PCI device ID: , UID: "660b8a9b-8a30-4894-a806-ffcf80bc53ee", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0327 22:59:57.634278 3898 cni.go:424] [openshift-image-registry/image-registry-656f8959d9-s9w6v e712050acf8f2d0cea9874a06285c2a59e26d091a7e196e05c01d6e8e5560204 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-656f8959d9-s9w6v e712050acf8f2d0cea9874a06285c2a59e26d091a7e196e05c01d6e8e5560204 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:be:44:82:f8:29\",\"name\":\"e712050acf8f2d0\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4536a110-b000-4ed3-ada2-7950baf06c72\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 22:59:58.142135 3898 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 22:59:59.139619 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0327 22:59:59.164425 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0327 22:59:59.748082 3898 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-c56ff685-gqjrq] creating logical port openshift-monitoring_prometheus-operator-c56ff685-gqjrq for pod on switch ip-10-0-0-48.ec2.internal I0327 22:59:59.748182 3898 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-c56ff685-gqjrq I0327 22:59:59.760789 3898 pod.go:62] [openshift-monitoring/prometheus-operator-c56ff685-gqjrq] pod update took 12.635817ms I0327 22:59:59.760808 3898 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-c56ff685-gqjrq] addLogicalPort annotation time took 12.658298ms I0327 22:59:59.762084 3898 pods.go:271] [openshift-monitoring/prometheus-operator-c56ff685-gqjrq] addLogicalPort took 14.017392ms, libovsdb time 909.027µs I0327 23:00:00.111846 3898 cni.go:403] [openshift-monitoring/prometheus-operator-c56ff685-gqjrq 65992b3deaffc3e3cb6567d2684480783a915225aee57016349fb30948d56e17 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-c56ff685-gqjrq 65992b3deaffc3e3cb6567d2684480783a915225aee57016349fb30948d56e17 network default NAD default] I0327 23:00:00.120218 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-c56ff685-gqjrq, hostIfaceName: 65992b3deaffc3e, network: default, NAD default, SandboxID: "65992b3deaffc3e3cb6567d2684480783a915225aee57016349fb30948d56e17", PCI device ID: , UID: "341cf4ce-6f79-4c7d-81b4-b47303695bb0", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0327 23:00:00.160871 3898 cni.go:424] [openshift-monitoring/prometheus-operator-c56ff685-gqjrq 65992b3deaffc3e3cb6567d2684480783a915225aee57016349fb30948d56e17 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-c56ff685-gqjrq 65992b3deaffc3e3cb6567d2684480783a915225aee57016349fb30948d56e17 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:c7:07:f2:46:74\",\"name\":\"65992b3deaffc3e\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/874b54e2-5f0c-4e96-9099-19f0058d9ddb\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:00:02.396050 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 23:00:02.396078 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0327 23:00:04.206553 3898 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-q9dc2] creating logical port openshift-monitoring_kube-state-metrics-664d8f7fc8-q9dc2 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:00:04.206742 3898 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-664d8f7fc8-q9dc2 I0327 23:00:04.226249 3898 pod.go:62] [openshift-monitoring/kube-state-metrics-664d8f7fc8-q9dc2] pod update took 19.58156ms I0327 23:00:04.226276 3898 base_network_controller_pods.go:937] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-q9dc2] addLogicalPort annotation time took 19.610615ms I0327 23:00:04.227492 3898 pods.go:271] [openshift-monitoring/kube-state-metrics-664d8f7fc8-q9dc2] addLogicalPort took 20.960263ms, libovsdb time 878.896µs I0327 23:00:04.564342 3898 cni.go:403] [openshift-monitoring/kube-state-metrics-664d8f7fc8-q9dc2 872b1c2fa5ead737ec58cff5567faf7d56de2041302aeffd7cde2a673e76d90a network default NAD default] ADD starting CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-q9dc2 872b1c2fa5ead737ec58cff5567faf7d56de2041302aeffd7cde2a673e76d90a network default NAD default] I0327 23:00:04.572889 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-664d8f7fc8-q9dc2, hostIfaceName: 872b1c2fa5ead73, network: default, NAD default, SandboxID: "872b1c2fa5ead737ec58cff5567faf7d56de2041302aeffd7cde2a673e76d90a", PCI device ID: , UID: "3207bebf-6613-4191-8b08-8ff7739ffa0b", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0327 23:00:04.617263 3898 cni.go:424] [openshift-monitoring/kube-state-metrics-664d8f7fc8-q9dc2 872b1c2fa5ead737ec58cff5567faf7d56de2041302aeffd7cde2a673e76d90a network default NAD default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-q9dc2 872b1c2fa5ead737ec58cff5567faf7d56de2041302aeffd7cde2a673e76d90a network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:36:72:88:30:5d\",\"name\":\"872b1c2fa5ead73\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0af7e8f6-b25a-412c-8b52-85a86ddaf4a5\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:00:04.937730 3898 cni.go:403] [openshift-dns/dns-default-kzdgm 5c8a7593a8aba5e1506209ee047ae2a015b691c3349a728d8192bc402f720713 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-kzdgm 5c8a7593a8aba5e1506209ee047ae2a015b691c3349a728d8192bc402f720713 network default NAD default] I0327 23:00:04.946613 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-kzdgm, hostIfaceName: 5c8a7593a8aba5e, network: default, NAD default, SandboxID: "5c8a7593a8aba5e1506209ee047ae2a015b691c3349a728d8192bc402f720713", PCI device ID: , UID: "eb7d70bf-860b-4df2-bb7f-732af80d446f", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0327 23:00:04.988567 3898 cni.go:424] [openshift-dns/dns-default-kzdgm 5c8a7593a8aba5e1506209ee047ae2a015b691c3349a728d8192bc402f720713 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-kzdgm 5c8a7593a8aba5e1506209ee047ae2a015b691c3349a728d8192bc402f720713 network default NAD default], result "{\"interfaces\":[{\"mac\":\"76:90:48:9d:5c:87\",\"name\":\"5c8a7593a8aba5e\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c6e3f55f-25b4-409b-b03b-b4c7bedf56a8\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:00:05.331452 3898 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:00:05.331589 3898 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0327 23:00:05.353305 3898 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 21.765575ms I0327 23:00:05.353330 3898 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 21.79541ms I0327 23:00:05.354642 3898 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 23.20688ms, libovsdb time 902.594µs I0327 23:00:06.753071 3898 cni.go:403] [openshift-monitoring/alertmanager-main-1 f5b0ad9be823971346b29fb67eea0d5c6474fa3e18bb6e098bdf61d32d5a1d7c network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 f5b0ad9be823971346b29fb67eea0d5c6474fa3e18bb6e098bdf61d32d5a1d7c network default NAD default] I0327 23:00:06.764951 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: f5b0ad9be823971, network: default, NAD default, SandboxID: "f5b0ad9be823971346b29fb67eea0d5c6474fa3e18bb6e098bdf61d32d5a1d7c", PCI device ID: , UID: "5ba8fde8-ada6-412a-ad5a-f01695640bc4", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0327 23:00:06.824427 3898 cni.go:424] [openshift-monitoring/alertmanager-main-1 f5b0ad9be823971346b29fb67eea0d5c6474fa3e18bb6e098bdf61d32d5a1d7c network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 f5b0ad9be823971346b29fb67eea0d5c6474fa3e18bb6e098bdf61d32d5a1d7c network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:6f:b2:6c:d7:89\",\"name\":\"f5b0ad9be823971\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/39c231a2-743a-480a-bbfa-30beaaf15203\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:00:07.053511 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0327 23:00:07.578839 3898 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:07.578859 3898 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:07.578866 3898 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 23:00:08.943364 3898 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-zpdk7] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-zpdk7 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:00:08.943500 3898 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-zpdk7 I0327 23:00:08.960109 3898 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-zpdk7] pod update took 16.658523ms I0327 23:00:08.960139 3898 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-zpdk7] addLogicalPort annotation time took 16.694543ms I0327 23:00:08.961437 3898 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-zpdk7] addLogicalPort took 18.089411ms, libovsdb time 930.945µs W0327 23:00:09.213440 3898 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:09.213482 3898 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:09.213491 3898 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 23:00:09.303915 3898 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-zpdk7 e87f5583c8723db628ff96a1d0144870c168e8151728781b3ed75cf70d85135a network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-zpdk7 e87f5583c8723db628ff96a1d0144870c168e8151728781b3ed75cf70d85135a network default NAD default] I0327 23:00:09.312646 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-zpdk7, hostIfaceName: e87f5583c8723db, network: default, NAD default, SandboxID: "e87f5583c8723db628ff96a1d0144870c168e8151728781b3ed75cf70d85135a", PCI device ID: , UID: "7541e0c1-b625-4296-9f13-7307cdcd78fa", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0327 23:00:09.358265 3898 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-zpdk7 e87f5583c8723db628ff96a1d0144870c168e8151728781b3ed75cf70d85135a network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-zpdk7 e87f5583c8723db628ff96a1d0144870c168e8151728781b3ed75cf70d85135a network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:f9:aa:3c:26:dd\",\"name\":\"e87f5583c8723db\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dddd3cb3-0c92-44af-93aa-7ca92183f9b5\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:00:10.512978 3898 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.513005 3898 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.513012 3898 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.513019 3898 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.649506 3898 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.649528 3898 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.649537 3898 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:10.649543 3898 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 23:00:11.216401 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0327 23:00:11.237817 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0327 23:00:11.512844 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 23:00:11.568146 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 23:00:11.609483 3898 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:00:11.609506 3898 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:00:12.529222 3898 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:00:12.529250 3898 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 23:00:15.332367 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0327 23:00:15.594500 3898 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-ghwmt] creating logical port openshift-console_downloads-56d45c498b-ghwmt for pod on switch ip-10-0-0-48.ec2.internal I0327 23:00:15.594606 3898 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-ghwmt I0327 23:00:15.616515 3898 pod.go:62] [openshift-console/downloads-56d45c498b-ghwmt] pod update took 21.931732ms I0327 23:00:15.616543 3898 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-ghwmt] addLogicalPort annotation time took 21.966923ms I0327 23:00:15.617838 3898 pods.go:271] [openshift-console/downloads-56d45c498b-ghwmt] addLogicalPort took 23.356712ms, libovsdb time 898.201µs I0327 23:00:15.953954 3898 cni.go:403] [openshift-console/downloads-56d45c498b-ghwmt 31633bf543b5df228be81d63c00dd95222391246632ebab65843d24d888c8661 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-ghwmt 31633bf543b5df228be81d63c00dd95222391246632ebab65843d24d888c8661 network default NAD default] I0327 23:00:15.962487 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-ghwmt, hostIfaceName: 31633bf543b5df2, network: default, NAD default, SandboxID: "31633bf543b5df228be81d63c00dd95222391246632ebab65843d24d888c8661", PCI device ID: , UID: "8d923e1f-fe3f-4a6f-9216-10a023083821", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0327 23:00:16.005687 3898 cni.go:424] [openshift-console/downloads-56d45c498b-ghwmt 31633bf543b5df228be81d63c00dd95222391246632ebab65843d24d888c8661 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-ghwmt 31633bf543b5df228be81d63c00dd95222391246632ebab65843d24d888c8661 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:48:31:1d:8c:a6\",\"name\":\"31633bf543b5df2\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9d35666-0836-460e-961a-710f947b1921\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:00:17.202285 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 23:00:17.202319 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 23:00:17.202325 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 23:00:18.487484 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 23:00:18.487518 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 23:00:18.525068 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 23:00:18.525092 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 23:00:19.152237 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0327 23:00:25.630449 3898 base_network_controller_pods.go:487] [default/openshift-console/console-8665c6c979-4sq4c] creating logical port openshift-console_console-8665c6c979-4sq4c for pod on switch ip-10-0-0-48.ec2.internal I0327 23:00:25.630613 3898 kube.go:256] Updating pod openshift-console/console-8665c6c979-4sq4c I0327 23:00:25.647670 3898 pod.go:62] [openshift-console/console-8665c6c979-4sq4c] pod update took 17.085597ms I0327 23:00:25.647692 3898 base_network_controller_pods.go:937] [default/openshift-console/console-8665c6c979-4sq4c] addLogicalPort annotation time took 17.113626ms I0327 23:00:25.648863 3898 pods.go:271] [openshift-console/console-8665c6c979-4sq4c] addLogicalPort took 18.432803ms, libovsdb time 846.058µs I0327 23:00:32.229789 3898 cni.go:403] [openshift-console/console-8665c6c979-4sq4c 5a4ea301799a1a55b5615938ead0df560898df03582b4ebd2ab8da4f543c2582 network default NAD default] ADD starting CNI request [openshift-console/console-8665c6c979-4sq4c 5a4ea301799a1a55b5615938ead0df560898df03582b4ebd2ab8da4f543c2582 network default NAD default] I0327 23:00:32.237987 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-8665c6c979-4sq4c, hostIfaceName: 5a4ea301799a1a5, network: default, NAD default, SandboxID: "5a4ea301799a1a55b5615938ead0df560898df03582b4ebd2ab8da4f543c2582", PCI device ID: , UID: "1e4ab3c6-fca2-447e-8afa-9b661589d334", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0327 23:00:32.282002 3898 cni.go:424] [openshift-console/console-8665c6c979-4sq4c 5a4ea301799a1a55b5615938ead0df560898df03582b4ebd2ab8da4f543c2582 network default NAD default] ADD finished CNI request [openshift-console/console-8665c6c979-4sq4c 5a4ea301799a1a55b5615938ead0df560898df03582b4ebd2ab8da4f543c2582 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:e8:aa:95:93:3c\",\"name\":\"5a4ea301799a1a5\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/92119844-6255-4569-b17d-2e85ccc2197f\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:00:32.605452 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0327 23:00:33.333273 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0327 23:00:35.621544 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0327 23:00:35.641820 3898 cni.go:403] [openshift-multus/network-metrics-daemon-n7l7j 43ac65a648536a60a706f7d1d2c61b040d4bf81d51c5c170449bbdb6536bef26 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-n7l7j 43ac65a648536a60a706f7d1d2c61b040d4bf81d51c5c170449bbdb6536bef26 network default NAD default] I0327 23:00:35.651967 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-n7l7j, hostIfaceName: 43ac65a648536a6, network: default, NAD default, SandboxID: "43ac65a648536a60a706f7d1d2c61b040d4bf81d51c5c170449bbdb6536bef26", PCI device ID: , UID: "def6963e-ef31-4ecd-b88d-cda66062778b", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0327 23:00:35.700724 3898 cni.go:424] [openshift-multus/network-metrics-daemon-n7l7j 43ac65a648536a60a706f7d1d2c61b040d4bf81d51c5c170449bbdb6536bef26 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-n7l7j 43ac65a648536a60a706f7d1d2c61b040d4bf81d51c5c170449bbdb6536bef26 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:e9:0c:48:71:07\",\"name\":\"43ac65a648536a6\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7ac5efab-9d03-45e6-baa5-6e45c0c3938c\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:00:35.749242 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:35.749271 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:35.749279 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:35.749298 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:35.749303 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:35.749308 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:35.802218 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:35.802243 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:35.802249 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:00:36.320579 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 23:00:36.635967 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 23:00:36.636000 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 23:00:43.360776 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 23:00:43.360785 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0327 23:00:43.360832 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0327 23:00:43.834377 3898 obj_retry.go:464] Detected object openshift-image-registry/image-registry-656cf7696d-mgnhg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:00:43.834926 3898 obj_retry.go:464] Detected object openshift-image-registry/image-registry-656cf7696d-mgnhg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 23:00:44.363252 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0327 23:00:44.651298 3898 obj_retry.go:458] Detected object openshift-image-registry/image-registry-656cf7696d-mgnhg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:00:44.651323 3898 obj_retry.go:458] Detected object openshift-image-registry/image-registry-656cf7696d-mgnhg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:00:44.657194 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-656cf7696d-mgnhg I0327 23:00:44.657216 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-656cf7696d-mgnhg W0327 23:00:44.735418 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 23:00:44.735447 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 23:00:44.768561 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 23:00:44.768590 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 23:00:53.403582 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 23:00:53.403651 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 23:00:53.403705 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 23:00:54.405670 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 23:00:59.435449 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0327 23:00:59.435639 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0327 23:00:59.435534 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0327 23:01:00.436595 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0327 23:01:01.487054 3898 cni.go:403] [openshift-console/console-8665c6c979-4sq4c 5a4ea301799a1a55b5615938ead0df560898df03582b4ebd2ab8da4f543c2582 network default NAD default] DEL starting CNI request [openshift-console/console-8665c6c979-4sq4c 5a4ea301799a1a55b5615938ead0df560898df03582b4ebd2ab8da4f543c2582 network default NAD default] I0327 23:01:01.564087 3898 cni.go:424] [openshift-console/console-8665c6c979-4sq4c 5a4ea301799a1a55b5615938ead0df560898df03582b4ebd2ab8da4f543c2582 network default NAD default] DEL finished CNI request [openshift-console/console-8665c6c979-4sq4c 5a4ea301799a1a55b5615938ead0df560898df03582b4ebd2ab8da4f543c2582 network default NAD default], result "{}", err I0327 23:01:01.652155 3898 obj_retry.go:464] Detected object openshift-console/console-8665c6c979-4sq4c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:01.652199 3898 pods.go:176] Deleting pod: openshift-console/console-8665c6c979-4sq4c I0327 23:01:01.653598 3898 pods.go:236] Attempting to release IPs for pod: openshift-console/console-8665c6c979-4sq4c, ips: 10.130.0.24 I0327 23:01:01.653635 3898 obj_retry.go:464] Detected object openshift-console/console-8665c6c979-4sq4c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:01.661390 3898 base_network_controller_pods.go:487] [default/openshift-console/console-759495f8d9-2zdq6] creating logical port openshift-console_console-759495f8d9-2zdq6 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:01:01.661565 3898 kube.go:256] Updating pod openshift-console/console-759495f8d9-2zdq6 I0327 23:01:01.680203 3898 pod.go:62] [openshift-console/console-759495f8d9-2zdq6] pod update took 18.662979ms I0327 23:01:01.680234 3898 base_network_controller_pods.go:937] [default/openshift-console/console-759495f8d9-2zdq6] addLogicalPort annotation time took 18.69866ms I0327 23:01:01.681797 3898 pods.go:271] [openshift-console/console-759495f8d9-2zdq6] addLogicalPort took 20.423878ms, libovsdb time 1.056345ms I0327 23:01:02.072353 3898 cni.go:403] [openshift-console/console-759495f8d9-2zdq6 63b4f2e21007ed96f82709f35600a7db6c015695b5b0088cc1611c41dd33168a network default NAD default] ADD starting CNI request [openshift-console/console-759495f8d9-2zdq6 63b4f2e21007ed96f82709f35600a7db6c015695b5b0088cc1611c41dd33168a network default NAD default] I0327 23:01:02.080630 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-759495f8d9-2zdq6, hostIfaceName: 63b4f2e21007ed9, network: default, NAD default, SandboxID: "63b4f2e21007ed96f82709f35600a7db6c015695b5b0088cc1611c41dd33168a", PCI device ID: , UID: "4f681177-c481-4e12-890e-8485afec1303", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0327 23:01:02.124297 3898 cni.go:424] [openshift-console/console-759495f8d9-2zdq6 63b4f2e21007ed96f82709f35600a7db6c015695b5b0088cc1611c41dd33168a network default NAD default] ADD finished CNI request [openshift-console/console-759495f8d9-2zdq6 63b4f2e21007ed96f82709f35600a7db6c015695b5b0088cc1611c41dd33168a network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:26:7d:55:f8:40\",\"name\":\"63b4f2e21007ed9\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/533d3a0b-fa39-4a3b-b31b-db38b972312c\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:01:02.457211 3898 obj_retry.go:458] Detected object openshift-console/console-8665c6c979-4sq4c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:02.457236 3898 obj_retry.go:458] Detected object openshift-console/console-8665c6c979-4sq4c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:02.460226 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8665c6c979-4sq4c I0327 23:01:02.460257 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8665c6c979-4sq4c I0327 23:01:10.084665 3898 obj_retry.go:464] Detected object openshift-console/console-8665c6c979-b5t6s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:10.085227 3898 obj_retry.go:464] Detected object openshift-console/console-8665c6c979-b5t6s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:10.752871 3898 obj_retry.go:458] Detected object openshift-console/console-8665c6c979-b5t6s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:10.752900 3898 obj_retry.go:458] Detected object openshift-console/console-8665c6c979-b5t6s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:10.758783 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8665c6c979-b5t6s I0327 23:01:10.758806 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8665c6c979-b5t6s I0327 23:01:11.400895 3898 obj_retry.go:464] Detected object openshift-ingress/router-default-85bf9b9667-mm9hl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:11.401407 3898 obj_retry.go:464] Detected object openshift-ingress/router-default-85bf9b9667-mm9hl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:11.758285 3898 obj_retry.go:458] Detected object openshift-ingress/router-default-85bf9b9667-mm9hl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:11.758311 3898 obj_retry.go:458] Detected object openshift-ingress/router-default-85bf9b9667-mm9hl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:11.763956 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-85bf9b9667-mm9hl I0327 23:01:11.763978 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-85bf9b9667-mm9hl W0327 23:01:12.496897 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 23:01:25.843200 3898 cni.go:403] [openshift-monitoring/alertmanager-main-1 f5b0ad9be823971346b29fb67eea0d5c6474fa3e18bb6e098bdf61d32d5a1d7c network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 f5b0ad9be823971346b29fb67eea0d5c6474fa3e18bb6e098bdf61d32d5a1d7c network default NAD default] I0327 23:01:25.927962 3898 cni.go:424] [openshift-monitoring/alertmanager-main-1 f5b0ad9be823971346b29fb67eea0d5c6474fa3e18bb6e098bdf61d32d5a1d7c network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 f5b0ad9be823971346b29fb67eea0d5c6474fa3e18bb6e098bdf61d32d5a1d7c network default NAD default], result "{}", err I0327 23:01:25.986725 3898 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:25.986766 3898 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0327 23:01:25.988257 3898 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.130.0.21 I0327 23:01:25.988288 3898 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:26.549807 3898 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:26.549832 3898 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:26.556595 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0327 23:01:26.556624 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0327 23:01:26.593954 3898 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:01:26.594057 3898 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0327 23:01:26.617552 3898 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 23.537385ms I0327 23:01:26.617574 3898 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 23.563098ms I0327 23:01:26.618681 3898 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 24.747544ms, libovsdb time 796.963µs I0327 23:01:26.966534 3898 cni.go:403] [openshift-monitoring/alertmanager-main-1 b1b669519f6a0a35817b6eebf63bd5300e7b7d0561c9e270dc16cb86abcc0073 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 b1b669519f6a0a35817b6eebf63bd5300e7b7d0561c9e270dc16cb86abcc0073 network default NAD default] I0327 23:01:26.975150 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: b1b669519f6a0a3, network: default, NAD default, SandboxID: "b1b669519f6a0a35817b6eebf63bd5300e7b7d0561c9e270dc16cb86abcc0073", PCI device ID: , UID: "223c0114-f438-4721-9172-f52076d5537f", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0327 23:01:27.020157 3898 cni.go:424] [openshift-monitoring/alertmanager-main-1 b1b669519f6a0a35817b6eebf63bd5300e7b7d0561c9e270dc16cb86abcc0073 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 b1b669519f6a0a35817b6eebf63bd5300e7b7d0561c9e270dc16cb86abcc0073 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:f6:b2:f7:1f:39\",\"name\":\"b1b669519f6a0a3\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/965227d7-390b-4efc-9166-8db1835a4bfc\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:01:28.761287 3898 base_network_controller_pods.go:487] [default/openshift-monitoring/telemeter-client-7c6c646889-vxmfz] creating logical port openshift-monitoring_telemeter-client-7c6c646889-vxmfz for pod on switch ip-10-0-0-48.ec2.internal I0327 23:01:28.761429 3898 kube.go:256] Updating pod openshift-monitoring/telemeter-client-7c6c646889-vxmfz I0327 23:01:28.779183 3898 pod.go:62] [openshift-monitoring/telemeter-client-7c6c646889-vxmfz] pod update took 17.785571ms I0327 23:01:28.779208 3898 base_network_controller_pods.go:937] [default/openshift-monitoring/telemeter-client-7c6c646889-vxmfz] addLogicalPort annotation time took 17.815198ms I0327 23:01:28.780599 3898 pods.go:271] [openshift-monitoring/telemeter-client-7c6c646889-vxmfz] addLogicalPort took 19.327257ms, libovsdb time 1.034331ms I0327 23:01:29.128873 3898 cni.go:403] [openshift-monitoring/telemeter-client-7c6c646889-vxmfz 588c1757a0742b7cab718448b65b395be5f2fa197ca2ba4572c1ddb496c9ed86 network default NAD default] ADD starting CNI request [openshift-monitoring/telemeter-client-7c6c646889-vxmfz 588c1757a0742b7cab718448b65b395be5f2fa197ca2ba4572c1ddb496c9ed86 network default NAD default] I0327 23:01:29.137888 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: telemeter-client-7c6c646889-vxmfz, hostIfaceName: 588c1757a0742b7, network: default, NAD default, SandboxID: "588c1757a0742b7cab718448b65b395be5f2fa197ca2ba4572c1ddb496c9ed86", PCI device ID: , UID: "2fc38363-0bba-4658-a94d-79f311bf944a", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0327 23:01:29.184142 3898 cni.go:424] [openshift-monitoring/telemeter-client-7c6c646889-vxmfz 588c1757a0742b7cab718448b65b395be5f2fa197ca2ba4572c1ddb496c9ed86 network default NAD default] ADD finished CNI request [openshift-monitoring/telemeter-client-7c6c646889-vxmfz 588c1757a0742b7cab718448b65b395be5f2fa197ca2ba4572c1ddb496c9ed86 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:00:76:18:59:4d\",\"name\":\"588c1757a0742b7\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2a9eb600-679f-4c20-ad6e-ed3b7152f79a\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:01:30.369824 3898 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:30.370381 3898 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:30.901938 3898 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:30.901984 3898 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:30.908405 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0327 23:01:30.908435 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0327 23:01:36.058073 3898 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-7mjmh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:36.058650 3898 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-7mjmh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:37.092956 3898 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-7mjmh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:37.092985 3898 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577540-7mjmh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 23:01:57.013990 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:01:57.014017 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 23:01:57.014023 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0327 23:01:57.047266 3898 cni.go:403] [openshift-console/console-759495f8d9-2zdq6 63b4f2e21007ed96f82709f35600a7db6c015695b5b0088cc1611c41dd33168a network default NAD default] DEL starting CNI request [openshift-console/console-759495f8d9-2zdq6 63b4f2e21007ed96f82709f35600a7db6c015695b5b0088cc1611c41dd33168a network default NAD default] I0327 23:01:57.129007 3898 cni.go:424] [openshift-console/console-759495f8d9-2zdq6 63b4f2e21007ed96f82709f35600a7db6c015695b5b0088cc1611c41dd33168a network default NAD default] DEL finished CNI request [openshift-console/console-759495f8d9-2zdq6 63b4f2e21007ed96f82709f35600a7db6c015695b5b0088cc1611c41dd33168a network default NAD default], result "{}", err I0327 23:01:57.174080 3898 obj_retry.go:464] Detected object openshift-console/console-759495f8d9-2zdq6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:57.174115 3898 pods.go:176] Deleting pod: openshift-console/console-759495f8d9-2zdq6 I0327 23:01:57.175356 3898 pods.go:236] Attempting to release IPs for pod: openshift-console/console-759495f8d9-2zdq6, ips: 10.130.0.25 I0327 23:01:57.175383 3898 obj_retry.go:464] Detected object openshift-console/console-759495f8d9-2zdq6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:57.182736 3898 base_network_controller_pods.go:487] [default/openshift-console/console-84546c86b9-j852z] creating logical port openshift-console_console-84546c86b9-j852z for pod on switch ip-10-0-0-48.ec2.internal I0327 23:01:57.182817 3898 kube.go:256] Updating pod openshift-console/console-84546c86b9-j852z I0327 23:01:57.196104 3898 pod.go:62] [openshift-console/console-84546c86b9-j852z] pod update took 13.308283ms I0327 23:01:57.196130 3898 base_network_controller_pods.go:937] [default/openshift-console/console-84546c86b9-j852z] addLogicalPort annotation time took 13.338536ms I0327 23:01:57.197366 3898 pods.go:271] [openshift-console/console-84546c86b9-j852z] addLogicalPort took 14.640804ms, libovsdb time 821.692µs I0327 23:01:57.538845 3898 cni.go:403] [openshift-console/console-84546c86b9-j852z ff2837ea0f1c679063ee2e45728a974030b99927cc3528b3f8d34468ebd16232 network default NAD default] ADD starting CNI request [openshift-console/console-84546c86b9-j852z ff2837ea0f1c679063ee2e45728a974030b99927cc3528b3f8d34468ebd16232 network default NAD default] I0327 23:01:57.548141 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-84546c86b9-j852z, hostIfaceName: ff2837ea0f1c679, network: default, NAD default, SandboxID: "ff2837ea0f1c679063ee2e45728a974030b99927cc3528b3f8d34468ebd16232", PCI device ID: , UID: "2aeb66c5-4821-41e1-b462-b9499edbabf0", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0327 23:01:57.684447 3898 obj_retry.go:458] Detected object openshift-console/console-759495f8d9-2zdq6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:57.684490 3898 obj_retry.go:458] Detected object openshift-console/console-759495f8d9-2zdq6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:57.690753 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-759495f8d9-2zdq6 I0327 23:01:57.690781 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-759495f8d9-2zdq6 I0327 23:01:57.798235 3898 cni.go:424] [openshift-console/console-84546c86b9-j852z ff2837ea0f1c679063ee2e45728a974030b99927cc3528b3f8d34468ebd16232 network default NAD default] ADD finished CNI request [openshift-console/console-84546c86b9-j852z ff2837ea0f1c679063ee2e45728a974030b99927cc3528b3f8d34468ebd16232 network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:b2:25:7f:7f:a2\",\"name\":\"ff2837ea0f1c679\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9761c0f8-6d3a-4aa8-9490-ede887d3fbd3\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:01:58.315014 3898 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:58.315584 3898 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:01:58.800630 3898 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 23:01:59.040424 3898 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:59.040452 3898 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:01:59.046752 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0327 23:01:59.046776 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 W0327 23:02:07.745525 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 23:02:08.243484 3898 obj_retry.go:464] Detected object openshift-console/console-759495f8d9-x5c28 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:08.243981 3898 obj_retry.go:464] Detected object openshift-console/console-759495f8d9-x5c28 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:09.086957 3898 obj_retry.go:458] Detected object openshift-console/console-759495f8d9-x5c28 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:09.086981 3898 obj_retry.go:458] Detected object openshift-console/console-759495f8d9-x5c28 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:09.089306 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-759495f8d9-x5c28 I0327 23:02:09.089336 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-759495f8d9-x5c28 I0327 23:02:33.528340 3898 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:33.529549 3898 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:02:34.074400 3898 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:34.074421 3898 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:02:34.080560 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0327 23:02:34.080582 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0327 23:04:02.661067 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0327 23:04:04.293833 3898 cni.go:403] [openshift-console/console-84546c86b9-j852z ff2837ea0f1c679063ee2e45728a974030b99927cc3528b3f8d34468ebd16232 network default NAD default] DEL starting CNI request [openshift-console/console-84546c86b9-j852z ff2837ea0f1c679063ee2e45728a974030b99927cc3528b3f8d34468ebd16232 network default NAD default] I0327 23:04:04.371587 3898 cni.go:424] [openshift-console/console-84546c86b9-j852z ff2837ea0f1c679063ee2e45728a974030b99927cc3528b3f8d34468ebd16232 network default NAD default] DEL finished CNI request [openshift-console/console-84546c86b9-j852z ff2837ea0f1c679063ee2e45728a974030b99927cc3528b3f8d34468ebd16232 network default NAD default], result "{}", err I0327 23:04:04.416813 3898 obj_retry.go:464] Detected object openshift-console/console-84546c86b9-j852z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:04.416851 3898 pods.go:176] Deleting pod: openshift-console/console-84546c86b9-j852z I0327 23:04:04.418062 3898 pods.go:236] Attempting to release IPs for pod: openshift-console/console-84546c86b9-j852z, ips: 10.130.0.28 I0327 23:04:04.418092 3898 obj_retry.go:464] Detected object openshift-console/console-84546c86b9-j852z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:04.425530 3898 base_network_controller_pods.go:487] [default/openshift-console/console-749db68589-hr6w2] creating logical port openshift-console_console-749db68589-hr6w2 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:04:04.425734 3898 kube.go:256] Updating pod openshift-console/console-749db68589-hr6w2 I0327 23:04:04.452929 3898 pod.go:62] [openshift-console/console-749db68589-hr6w2] pod update took 27.234554ms I0327 23:04:04.452952 3898 base_network_controller_pods.go:937] [default/openshift-console/console-749db68589-hr6w2] addLogicalPort annotation time took 27.263108ms I0327 23:04:04.454280 3898 pods.go:271] [openshift-console/console-749db68589-hr6w2] addLogicalPort took 28.765523ms, libovsdb time 880.432µs I0327 23:04:04.784495 3898 cni.go:403] [openshift-console/console-749db68589-hr6w2 34e637a099ebe3cd1fa253060feacfde7c16643949d0dc409c60b2cd28fb5fef network default NAD default] ADD starting CNI request [openshift-console/console-749db68589-hr6w2 34e637a099ebe3cd1fa253060feacfde7c16643949d0dc409c60b2cd28fb5fef network default NAD default] I0327 23:04:04.792984 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-749db68589-hr6w2, hostIfaceName: 34e637a099ebe3c, network: default, NAD default, SandboxID: "34e637a099ebe3cd1fa253060feacfde7c16643949d0dc409c60b2cd28fb5fef", PCI device ID: , UID: "c6e81f0d-4a33-4251-82dd-b34892670e09", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] I0327 23:04:04.836261 3898 cni.go:424] [openshift-console/console-749db68589-hr6w2 34e637a099ebe3cd1fa253060feacfde7c16643949d0dc409c60b2cd28fb5fef network default NAD default] ADD finished CNI request [openshift-console/console-749db68589-hr6w2 34e637a099ebe3cd1fa253060feacfde7c16643949d0dc409c60b2cd28fb5fef network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:e6:af:27:35:77\",\"name\":\"34e637a099ebe3c\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a65dcd4d-1d35-4d90-b884-12fc0707d026\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:04:05.194582 3898 obj_retry.go:458] Detected object openshift-console/console-84546c86b9-j852z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:05.194611 3898 obj_retry.go:458] Detected object openshift-console/console-84546c86b9-j852z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:05.200330 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84546c86b9-j852z I0327 23:04:05.200355 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84546c86b9-j852z I0327 23:04:08.360049 3898 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cg4zd] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-cg4zd for pod on switch ip-10-0-0-48.ec2.internal I0327 23:04:08.360184 3898 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cg4zd I0327 23:04:08.375829 3898 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cg4zd] pod update took 15.671278ms I0327 23:04:08.375860 3898 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cg4zd] addLogicalPort annotation time took 15.705911ms I0327 23:04:08.377166 3898 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cg4zd] addLogicalPort took 17.138347ms, libovsdb time 912.954µs I0327 23:04:08.721293 3898 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cg4zd 56ebfaa63283749c5204200651743df33f0cce41105a099f75da545d2af6bf11 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cg4zd 56ebfaa63283749c5204200651743df33f0cce41105a099f75da545d2af6bf11 network default NAD default] I0327 23:04:08.730545 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-cg4zd, hostIfaceName: 56ebfaa63283749, network: default, NAD default, SandboxID: "56ebfaa63283749c5204200651743df33f0cce41105a099f75da545d2af6bf11", PCI device ID: , UID: "89e7e788-275a-4d8d-aac5-56742e047b58", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0327 23:04:08.982515 3898 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cg4zd 56ebfaa63283749c5204200651743df33f0cce41105a099f75da545d2af6bf11 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-cg4zd 56ebfaa63283749c5204200651743df33f0cce41105a099f75da545d2af6bf11 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:f8:9d:b8:ee:d2\",\"name\":\"56ebfaa63283749\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c8e93cec-9d5b-4337-9308-0e5b9f0e8aae\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:04:11.243932 3898 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 23:04:14.811750 3898 obj_retry.go:464] Detected object openshift-console/console-84546c86b9-jkpqt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:04:14.812277 3898 obj_retry.go:464] Detected object openshift-console/console-84546c86b9-jkpqt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 23:04:15.251212 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 23:04:15.458258 3898 obj_retry.go:458] Detected object openshift-console/console-84546c86b9-jkpqt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:15.458281 3898 obj_retry.go:458] Detected object openshift-console/console-84546c86b9-jkpqt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:04:15.463970 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-84546c86b9-jkpqt I0327 23:04:15.463988 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-84546c86b9-jkpqt I0327 23:04:20.764792 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 W0327 23:04:22.239599 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 23:04:22.270583 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0327 23:04:22.270651 3898 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-28bsv IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-28bsv: no pod IPs found I0327 23:04:29.781845 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0327 23:04:32.782330 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0327 23:04:36.777498 3898 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0327 23:04:43.771067 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0327 23:04:49.743137 3898 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0327 23:05:31.748330 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0327 23:05:54.754227 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 23:06:06.063412 3898 namespace.go:98] [tsf] adding namespace I0327 23:06:06.064422 3898 namespace.go:102] [tsf] adding namespace took 981.492µs I0327 23:06:06.078068 3898 namespace.go:142] [tsf] updating namespace I0327 23:06:06.089221 3898 namespace.go:142] [tsf] updating namespace I0327 23:06:09.797424 3898 namespace.go:98] [rhtpa-operator] adding namespace I0327 23:06:09.798259 3898 namespace.go:98] [konflux-ui] adding namespace I0327 23:06:09.798445 3898 namespace.go:102] [rhtpa-operator] adding namespace took 997.65µs I0327 23:06:09.799101 3898 namespace.go:102] [konflux-ui] adding namespace took 824.386µs I0327 23:06:09.801963 3898 namespace.go:98] [tsf-keycloak] adding namespace I0327 23:06:09.801999 3898 namespace.go:98] [cert-manager-operator] adding namespace I0327 23:06:09.802120 3898 namespace.go:98] [tsf-tas] adding namespace I0327 23:06:09.802162 3898 namespace.go:98] [konflux-operator] adding namespace I0327 23:06:09.802637 3898 namespace.go:102] [tsf-keycloak] adding namespace took 659.741µs I0327 23:06:09.803167 3898 namespace.go:102] [cert-manager-operator] adding namespace took 1.152948ms I0327 23:06:09.803646 3898 namespace.go:102] [tsf-tas] adding namespace took 1.511381ms I0327 23:06:09.804161 3898 namespace.go:102] [konflux-operator] adding namespace took 1.989709ms I0327 23:06:09.804701 3898 namespace.go:98] [rhbk-operator] adding namespace I0327 23:06:09.804779 3898 namespace.go:98] [tsf-tpa] adding namespace I0327 23:06:09.805320 3898 namespace.go:102] [rhbk-operator] adding namespace took 603.066µs I0327 23:06:09.805880 3898 namespace.go:102] [tsf-tpa] adding namespace took 1.088017ms I0327 23:06:09.817176 3898 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:06:09.827414 3898 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:06:09.846499 3898 namespace.go:142] [konflux-ui] updating namespace I0327 23:06:09.854562 3898 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:06:09.859762 3898 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:06:09.863651 3898 namespace.go:142] [tsf-tas] updating namespace I0327 23:06:09.872619 3898 namespace.go:142] [konflux-ui] updating namespace I0327 23:06:09.879342 3898 namespace.go:142] [konflux-operator] updating namespace I0327 23:06:09.883583 3898 namespace.go:142] [rhbk-operator] updating namespace I0327 23:06:09.886007 3898 namespace.go:142] [tsf-tpa] updating namespace I0327 23:06:09.888963 3898 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:06:09.946921 3898 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:06:09.966120 3898 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:06:09.978071 3898 namespace.go:142] [tsf-tas] updating namespace I0327 23:06:09.997580 3898 namespace.go:142] [tsf-tas] updating namespace I0327 23:06:10.005441 3898 namespace.go:142] [konflux-ui] updating namespace I0327 23:06:10.016282 3898 namespace.go:142] [konflux-operator] updating namespace I0327 23:06:10.028949 3898 namespace.go:142] [konflux-operator] updating namespace I0327 23:06:10.046417 3898 namespace.go:142] [rhbk-operator] updating namespace I0327 23:06:10.057449 3898 namespace.go:142] [rhbk-operator] updating namespace I0327 23:06:10.088274 3898 namespace.go:142] [tsf-tpa] updating namespace I0327 23:06:10.098659 3898 namespace.go:142] [tsf-tpa] updating namespace I0327 23:06:10.107490 3898 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:06:10.140412 3898 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:06:11.139317 3898 namespace.go:142] [tsf-tpa] updating namespace I0327 23:06:11.158987 3898 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:06:17.117691 3898 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:17.118267 3898 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:17.118294 3898 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:17.118301 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:17.118313 3898 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:17.118318 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:18.140968 3898 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:18.140997 3898 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:18.141012 3898 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:18.141023 3898 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba85ctx7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:21.074802 3898 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:21.075295 3898 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:21.075316 3898 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:21.075323 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:21.075332 3898 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:21.075336 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:22.099496 3898 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:22.099521 3898 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:22.099538 3898 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:22.099548 3898 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5z8kw9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:23.106155 3898 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:23.106682 3898 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:23.106709 3898 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:23.106717 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:23.106726 3898 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:23.106731 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:24.120689 3898 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.121223 3898 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.121248 3898 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.121256 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:24.121289 3898 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.121298 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:24.135676 3898 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:24.135696 3898 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:24.135710 3898 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:24.135722 3898 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfl5z9v of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:24.148218 3898 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.148666 3898 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.148701 3898 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.148713 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:24.148725 3898 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:24.148730 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:25.124376 3898 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:25.124917 3898 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:06:25.124950 3898 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:25.124960 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 23:06:25.124975 3898 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 23:06:25.124981 3898 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 23:06:25.147106 3898 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.147123 3898 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.147136 3898 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.147148 3898 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7j4qsg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.180352 3898 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.180371 3898 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.180380 3898 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.180390 3898 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1vlzdz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:25.531142 3898 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-g9lb6] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-g9lb6 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:06:25.531278 3898 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-g9lb6 I0327 23:06:25.556259 3898 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-g9lb6] pod update took 25.007605ms I0327 23:06:25.556280 3898 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-g9lb6] addLogicalPort annotation time took 25.033827ms I0327 23:06:25.557572 3898 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-g9lb6] addLogicalPort took 26.447291ms, libovsdb time 948.779µs I0327 23:06:25.887434 3898 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-g9lb6 d1970bdf2a7d4e003238c6c9ea6d68023902c97687d14893f0dca3ee3a616db4 network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-g9lb6 d1970bdf2a7d4e003238c6c9ea6d68023902c97687d14893f0dca3ee3a616db4 network default NAD default] I0327 23:06:25.896115 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-g9lb6, hostIfaceName: d1970bdf2a7d4e0, network: default, NAD default, SandboxID: "d1970bdf2a7d4e003238c6c9ea6d68023902c97687d14893f0dca3ee3a616db4", PCI device ID: , UID: "4d413df0-4d80-4b56-9e11-d29e122f4281", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0327 23:06:25.941224 3898 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-g9lb6 d1970bdf2a7d4e003238c6c9ea6d68023902c97687d14893f0dca3ee3a616db4 network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-g9lb6 d1970bdf2a7d4e003238c6c9ea6d68023902c97687d14893f0dca3ee3a616db4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:c1:e7:c3:db:8a\",\"name\":\"d1970bdf2a7d4e0\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be8f7f40-b1ff-443b-a9ff-d5febaf7afbb\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:06:26.147288 3898 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:26.147311 3898 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:26.147324 3898 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:06:26.147336 3898 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafm5flg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0327 23:06:28.826565 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0327 23:06:29.169605 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0327 23:06:29.178054 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 23:06:30.566753 3898 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gqljd] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-gqljd for pod on switch ip-10-0-0-48.ec2.internal I0327 23:06:30.566909 3898 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gqljd I0327 23:06:30.587582 3898 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gqljd] pod update took 20.705663ms I0327 23:06:30.587607 3898 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gqljd] addLogicalPort annotation time took 20.734588ms I0327 23:06:30.588980 3898 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gqljd] addLogicalPort took 22.246131ms, libovsdb time 1.007308ms I0327 23:06:30.931404 3898 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gqljd 50729eff10cc2deb3747a7f7b840cfab3c9c10ed98499eb975bf116f41ca29ad network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gqljd 50729eff10cc2deb3747a7f7b840cfab3c9c10ed98499eb975bf116f41ca29ad network default NAD default] I0327 23:06:30.941347 3898 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-gqljd, hostIfaceName: 50729eff10cc2de, network: default, NAD default, SandboxID: "50729eff10cc2deb3747a7f7b840cfab3c9c10ed98499eb975bf116f41ca29ad", PCI device ID: , UID: "b254b6a4-9d49-446d-96fe-15ab404d9781", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0327 23:06:30.985571 3898 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gqljd 50729eff10cc2deb3747a7f7b840cfab3c9c10ed98499eb975bf116f41ca29ad network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gqljd 50729eff10cc2deb3747a7f7b840cfab3c9c10ed98499eb975bf116f41ca29ad network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:60:14:a0:83:ce\",\"name\":\"50729eff10cc2de\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/69177115-1120-49ac-9b7d-cdcff79e1029\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:06:32.660562 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=73 I0327 23:06:37.789020 3898 namespace.go:98] [cert-manager] adding namespace I0327 23:06:37.789920 3898 namespace.go:102] [cert-manager] adding namespace took 873.596µs I0327 23:06:37.811164 3898 namespace.go:142] [cert-manager] updating namespace I0327 23:06:37.823992 3898 namespace.go:142] [cert-manager] updating namespace W0327 23:06:39.947339 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 23:06:40.503804 3898 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5b4bf68748-qp67q] creating logical port cert-manager_cert-manager-cainjector-5b4bf68748-qp67q for pod on switch ip-10-0-0-48.ec2.internal I0327 23:06:40.503905 3898 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5b4bf68748-qp67q I0327 23:06:40.520426 3898 pod.go:62] [cert-manager/cert-manager-cainjector-5b4bf68748-qp67q] pod update took 16.552094ms I0327 23:06:40.520449 3898 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-cainjector-5b4bf68748-qp67q] addLogicalPort annotation time took 16.580741ms I0327 23:06:40.521741 3898 pods.go:271] [cert-manager/cert-manager-cainjector-5b4bf68748-qp67q] addLogicalPort took 17.955307ms, libovsdb time 851.382µs I0327 23:06:40.874373 3898 cni.go:403] [cert-manager/cert-manager-cainjector-5b4bf68748-qp67q 08755c2fdb3244b5b4d2f23f826b41834302f20fbc1b97c0a98c878619f328f0 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-qp67q 08755c2fdb3244b5b4d2f23f826b41834302f20fbc1b97c0a98c878619f328f0 network default NAD default] I0327 23:06:40.882772 3898 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5b4bf68748-qp67q, hostIfaceName: 08755c2fdb3244b, network: default, NAD default, SandboxID: "08755c2fdb3244b5b4d2f23f826b41834302f20fbc1b97c0a98c878619f328f0", PCI device ID: , UID: "a7eca47a-df97-4e60-878f-49d0e45b421f", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0327 23:06:40.928100 3898 cni.go:424] [cert-manager/cert-manager-cainjector-5b4bf68748-qp67q 08755c2fdb3244b5b4d2f23f826b41834302f20fbc1b97c0a98c878619f328f0 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-qp67q 08755c2fdb3244b5b4d2f23f826b41834302f20fbc1b97c0a98c878619f328f0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:44:3f:a7:1e:b5\",\"name\":\"08755c2fdb3244b\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dfaf0f3c-e72e-4da1-9c79-4c226d3fd6f5\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:06:42.293240 3898 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-pss55] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-pss55 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:06:42.293429 3898 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 I0327 23:06:42.314631 3898 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55] pod update took 21.240157ms I0327 23:06:42.314663 3898 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-pss55] addLogicalPort annotation time took 21.276208ms I0327 23:06:42.316036 3898 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55] addLogicalPort took 22.820045ms, libovsdb time 880.575µs I0327 23:06:42.655123 3898 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 5663de9fc0cc7d78ef02a7486973a1cb55d18616b77a238b630bc607e87658cd network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 5663de9fc0cc7d78ef02a7486973a1cb55d18616b77a238b630bc607e87658cd network default NAD default] I0327 23:06:42.664372 3898 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-pss55, hostIfaceName: 5663de9fc0cc7d7, network: default, NAD default, SandboxID: "5663de9fc0cc7d78ef02a7486973a1cb55d18616b77a238b630bc607e87658cd", PCI device ID: , UID: "b0b7bc25-ddd8-4708-bc29-bf4d29922857", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0327 23:06:42.712454 3898 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 5663de9fc0cc7d78ef02a7486973a1cb55d18616b77a238b630bc607e87658cd network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 5663de9fc0cc7d78ef02a7486973a1cb55d18616b77a238b630bc607e87658cd network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:8a:6c:33:34:8b\",\"name\":\"5663de9fc0cc7d7\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/519607bb-1eda-42b0-b035-6eb99b1b50ae\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:06:43.480647 3898 namespace.go:142] [openshift-operators] updating namespace I0327 23:06:43.496602 3898 namespace.go:142] [openshift-operators] updating namespace W0327 23:06:45.928814 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 23:06:47.537294 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 23:06:47.537319 3898 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0327 23:06:48.309226 3898 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59ccb4bb6b-npc7b] creating logical port cert-manager_cert-manager-59ccb4bb6b-npc7b for pod on switch ip-10-0-0-48.ec2.internal I0327 23:06:48.309316 3898 kube.go:256] Updating pod cert-manager/cert-manager-59ccb4bb6b-npc7b I0327 23:06:48.354794 3898 pod.go:62] [cert-manager/cert-manager-59ccb4bb6b-npc7b] pod update took 45.500226ms I0327 23:06:48.354814 3898 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-59ccb4bb6b-npc7b] addLogicalPort annotation time took 45.526026ms I0327 23:06:48.356018 3898 pods.go:271] [cert-manager/cert-manager-59ccb4bb6b-npc7b] addLogicalPort took 46.805749ms, libovsdb time 858.18µs I0327 23:06:48.964881 3898 cni.go:403] [cert-manager/cert-manager-59ccb4bb6b-npc7b 23feff083f3d43b77d32d18fe38700272dccd5fa84aeed5e117c87221948142f network default NAD default] ADD starting CNI request [cert-manager/cert-manager-59ccb4bb6b-npc7b 23feff083f3d43b77d32d18fe38700272dccd5fa84aeed5e117c87221948142f network default NAD default] I0327 23:06:48.973304 3898 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59ccb4bb6b-npc7b, hostIfaceName: 23feff083f3d43b, network: default, NAD default, SandboxID: "23feff083f3d43b77d32d18fe38700272dccd5fa84aeed5e117c87221948142f", PCI device ID: , UID: "5ffee556-2bce-496f-8279-fcd0c520f00c", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0327 23:06:49.022286 3898 cni.go:424] [cert-manager/cert-manager-59ccb4bb6b-npc7b 23feff083f3d43b77d32d18fe38700272dccd5fa84aeed5e117c87221948142f network default NAD default] ADD finished CNI request [cert-manager/cert-manager-59ccb4bb6b-npc7b 23feff083f3d43b77d32d18fe38700272dccd5fa84aeed5e117c87221948142f network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:86:1c:40:31:16\",\"name\":\"23feff083f3d43b\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ffb6a618-78eb-4ac3-87ff-d3c2196c5813\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:06:49.791620 3898 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cv5sj] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-cv5sj for pod on switch ip-10-0-0-48.ec2.internal I0327 23:06:49.791758 3898 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cv5sj I0327 23:06:49.824449 3898 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cv5sj] pod update took 32.712914ms I0327 23:06:49.824483 3898 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cv5sj] addLogicalPort annotation time took 32.752576ms I0327 23:06:49.826068 3898 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cv5sj] addLogicalPort took 34.462501ms, libovsdb time 1.127451ms I0327 23:06:50.455486 3898 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cv5sj d944d334eb38f56abad0748d30f7bfaa917168ef50d14f8f5963461034cff15d network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cv5sj d944d334eb38f56abad0748d30f7bfaa917168ef50d14f8f5963461034cff15d network default NAD default] I0327 23:06:50.463786 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-cv5sj, hostIfaceName: d944d334eb38f56, network: default, NAD default, SandboxID: "d944d334eb38f56abad0748d30f7bfaa917168ef50d14f8f5963461034cff15d", PCI device ID: , UID: "2c134a9d-20f7-437c-8daa-2332de6fd3ee", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0327 23:06:50.507300 3898 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cv5sj d944d334eb38f56abad0748d30f7bfaa917168ef50d14f8f5963461034cff15d network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-cv5sj d944d334eb38f56abad0748d30f7bfaa917168ef50d14f8f5963461034cff15d network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:ae:a2:24:17:12\",\"name\":\"d944d334eb38f56\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ffe6a13-ed8f-471d-afc2-6533438289e9\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:06:53.186227 3898 namespace.go:98] [trusted-artifact-signer] adding namespace I0327 23:06:53.187115 3898 namespace.go:102] [trusted-artifact-signer] adding namespace took 863.168µs I0327 23:06:53.206042 3898 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:06:53.213841 3898 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:06:53.316938 3898 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-x7vj6] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-x7vj6 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:06:53.317056 3898 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-x7vj6 I0327 23:06:53.338979 3898 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-x7vj6] pod update took 21.943155ms I0327 23:06:53.339000 3898 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-x7vj6] addLogicalPort annotation time took 21.971942ms I0327 23:06:53.340313 3898 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-x7vj6] addLogicalPort took 23.401608ms, libovsdb time 968.581µs I0327 23:06:53.669716 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=150 I0327 23:06:53.976571 3898 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-x7vj6 ee63200654521a135dfca12a761b4debef8810b2c972f6ff85efe8df1d5d2021 network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-x7vj6 ee63200654521a135dfca12a761b4debef8810b2c972f6ff85efe8df1d5d2021 network default NAD default] W0327 23:06:53.981125 3898 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 23:06:53.985933 3898 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-x7vj6, hostIfaceName: ee63200654521a1, network: default, NAD default, SandboxID: "ee63200654521a135dfca12a761b4debef8810b2c972f6ff85efe8df1d5d2021", PCI device ID: , UID: "dfeb0e42-9d84-4d4a-ab24-ce037c1ac733", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0327 23:06:54.032185 3898 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-x7vj6 ee63200654521a135dfca12a761b4debef8810b2c972f6ff85efe8df1d5d2021 network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-x7vj6 ee63200654521a135dfca12a761b4debef8810b2c972f6ff85efe8df1d5d2021 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:69:d5:72:cc:5a\",\"name\":\"ee63200654521a1\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f29b1cd8-c6f1-4f0f-9733-e1f5c96991a6\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:06:56.260126 3898 namespace.go:142] [openshift-operators] updating namespace W0327 23:06:56.940060 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 23:07:04.973288 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 23:07:12.151683 3898 namespace.go:98] [openshift-pipelines] adding namespace I0327 23:07:12.152454 3898 namespace.go:102] [openshift-pipelines] adding namespace took 744.662µs I0327 23:07:12.169687 3898 namespace.go:142] [openshift-pipelines] updating namespace I0327 23:07:12.854562 3898 namespace.go:142] [cert-manager] updating namespace I0327 23:07:13.055843 3898 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:07:13.456729 3898 namespace.go:142] [konflux-operator] updating namespace I0327 23:07:13.656436 3898 namespace.go:142] [konflux-ui] updating namespace I0327 23:07:13.855733 3898 namespace.go:142] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] updating namespace I0327 23:07:14.260008 3898 namespace.go:142] [rhbk-operator] updating namespace I0327 23:07:14.462440 3898 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:07:14.662843 3898 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:07:14.863578 3898 namespace.go:142] [tsf] updating namespace I0327 23:07:15.064719 3898 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:07:15.265109 3898 namespace.go:142] [tsf-tas] updating namespace I0327 23:07:15.467433 3898 namespace.go:142] [tsf-tpa] updating namespace I0327 23:07:15.606276 3898 namespace.go:142] [cert-manager] updating namespace I0327 23:07:15.616292 3898 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:07:15.624339 3898 namespace.go:142] [default] updating namespace I0327 23:07:15.633751 3898 namespace.go:142] [konflux-operator] updating namespace I0327 23:07:15.642109 3898 namespace.go:142] [konflux-ui] updating namespace I0327 23:07:15.651790 3898 namespace.go:142] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] updating namespace I0327 23:07:15.660373 3898 namespace.go:142] [openshift] updating namespace I0327 23:07:15.671263 3898 namespace.go:142] [rhbk-operator] updating namespace I0327 23:07:15.679280 3898 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:07:15.689487 3898 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:07:15.698742 3898 namespace.go:142] [tsf] updating namespace I0327 23:07:15.710029 3898 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:07:15.731035 3898 namespace.go:142] [tsf-tas] updating namespace I0327 23:07:15.748820 3898 namespace.go:142] [tsf-tpa] updating namespace I0327 23:07:15.851891 3898 namespace.go:142] [cert-manager] updating namespace I0327 23:07:15.950740 3898 namespace.go:142] [cert-manager-operator] updating namespace I0327 23:07:16.049275 3898 namespace.go:142] [default] updating namespace I0327 23:07:16.154451 3898 namespace.go:142] [konflux-operator] updating namespace I0327 23:07:16.249677 3898 namespace.go:142] [konflux-ui] updating namespace I0327 23:07:16.355103 3898 namespace.go:142] [open-cluster-management-2pabjk1o0k4lirtjul81bm4bi3nfk4gj] updating namespace I0327 23:07:16.450384 3898 namespace.go:142] [openshift] updating namespace I0327 23:07:16.549538 3898 namespace.go:142] [rhbk-operator] updating namespace I0327 23:07:16.651699 3898 namespace.go:142] [rhtpa-operator] updating namespace I0327 23:07:16.751992 3898 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 23:07:16.758897 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0327 23:07:16.849031 3898 namespace.go:142] [tsf] updating namespace I0327 23:07:16.951152 3898 namespace.go:142] [tsf-keycloak] updating namespace I0327 23:07:17.051903 3898 namespace.go:142] [tsf-tas] updating namespace I0327 23:07:17.151770 3898 namespace.go:142] [tsf-tpa] updating namespace I0327 23:07:17.448541 3898 namespace.go:142] [openshift-pipelines] updating namespace I0327 23:07:23.410043 3898 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:07:23.410570 3898 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:07:23.430516 3898 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:07:23.430536 3898 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:07:23.436238 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0327 23:07:23.436257 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0327 23:07:26.179481 3898 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-pb7vc] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-pb7vc for pod on switch ip-10-0-0-48.ec2.internal I0327 23:07:26.179592 3898 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-pb7vc I0327 23:07:26.200139 3898 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-pb7vc] pod update took 20.580316ms I0327 23:07:26.200166 3898 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-pb7vc] addLogicalPort annotation time took 20.611689ms I0327 23:07:26.201577 3898 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-pb7vc] addLogicalPort took 22.125409ms, libovsdb time 936.904µs I0327 23:07:26.277386 3898 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-b79bdd59-s58bp] creating logical port openshift-pipelines_tekton-events-controller-b79bdd59-s58bp for pod on switch ip-10-0-0-48.ec2.internal I0327 23:07:26.277534 3898 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-b79bdd59-s58bp I0327 23:07:26.294002 3898 pod.go:62] [openshift-pipelines/tekton-events-controller-b79bdd59-s58bp] pod update took 16.496826ms I0327 23:07:26.294024 3898 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-events-controller-b79bdd59-s58bp] addLogicalPort annotation time took 16.525489ms I0327 23:07:26.295398 3898 pods.go:271] [openshift-pipelines/tekton-events-controller-b79bdd59-s58bp] addLogicalPort took 18.02904ms, libovsdb time 961.179µs I0327 23:07:26.379113 3898 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p for pod on switch ip-10-0-0-48.ec2.internal I0327 23:07:26.379229 3898 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p I0327 23:07:26.399217 3898 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p] pod update took 20.020602ms I0327 23:07:26.399246 3898 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p] addLogicalPort annotation time took 20.056012ms I0327 23:07:26.400771 3898 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p] addLogicalPort took 21.680434ms, libovsdb time 928.274µs I0327 23:07:26.543871 3898 cni.go:403] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-pb7vc 2e31207a0bd0d3c3a19dd74bf63e9a22bf64fe37e0750ec170038864575b5019 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-pb7vc 2e31207a0bd0d3c3a19dd74bf63e9a22bf64fe37e0750ec170038864575b5019 network default NAD default] I0327 23:07:26.552149 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-pb7vc, hostIfaceName: 2e31207a0bd0d3c, network: default, NAD default, SandboxID: "2e31207a0bd0d3c3a19dd74bf63e9a22bf64fe37e0750ec170038864575b5019", PCI device ID: , UID: "9d945528-6c2c-48c0-a516-707421d1d4b5", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0327 23:07:26.600543 3898 cni.go:424] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-pb7vc 2e31207a0bd0d3c3a19dd74bf63e9a22bf64fe37e0750ec170038864575b5019 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-pb7vc 2e31207a0bd0d3c3a19dd74bf63e9a22bf64fe37e0750ec170038864575b5019 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:45:3b:2f:51:1f\",\"name\":\"2e31207a0bd0d3c\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4e4689a5-0d60-4210-b285-1c4eaf151a5e\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:07:26.637700 3898 cni.go:403] [openshift-pipelines/tekton-events-controller-b79bdd59-s58bp 27e012b3ae1e53265131238e35816bb6a632b7a55595861baab4a13ca3293346 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-s58bp 27e012b3ae1e53265131238e35816bb6a632b7a55595861baab4a13ca3293346 network default NAD default] I0327 23:07:26.647391 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-b79bdd59-s58bp, hostIfaceName: 27e012b3ae1e532, network: default, NAD default, SandboxID: "27e012b3ae1e53265131238e35816bb6a632b7a55595861baab4a13ca3293346", PCI device ID: , UID: "18811a4e-0f67-44ad-99ec-e8696efe26a9", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0327 23:07:26.691686 3898 cni.go:424] [openshift-pipelines/tekton-events-controller-b79bdd59-s58bp 27e012b3ae1e53265131238e35816bb6a632b7a55595861baab4a13ca3293346 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-s58bp 27e012b3ae1e53265131238e35816bb6a632b7a55595861baab4a13ca3293346 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:9c:12:6c:4c:52\",\"name\":\"27e012b3ae1e532\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/34808b1b-d017-456b-b2db-9a4d78e39f67\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:07:26.753381 3898 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p 77419e1ff62f16667636a5725382a5cdf491d8df4e2f0119864cd18b658acd82 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p 77419e1ff62f16667636a5725382a5cdf491d8df4e2f0119864cd18b658acd82 network default NAD default] I0327 23:07:26.761895 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p, hostIfaceName: 77419e1ff62f166, network: default, NAD default, SandboxID: "77419e1ff62f16667636a5725382a5cdf491d8df4e2f0119864cd18b658acd82", PCI device ID: , UID: "4d0e3a8a-c4e0-4132-968f-d28fedf00f8a", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0327 23:07:26.811292 3898 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p 77419e1ff62f16667636a5725382a5cdf491d8df4e2f0119864cd18b658acd82 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-w8q9p 77419e1ff62f16667636a5725382a5cdf491d8df4e2f0119864cd18b658acd82 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:15:ca:43:a3:8f\",\"name\":\"77419e1ff62f166\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8408b161-aab2-40ff-b2a3-e784c434961a\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:07:30.473512 3898 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:07:30.473538 3898 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:07:30.473552 3898 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:07:30.473558 3898 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 23:07:31.255154 3898 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:07:31.255217 3898 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:07:31.255223 3898 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:07:31.255301 3898 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:07:31.255318 3898 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:07:31.255323 3898 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:07:38.371614 3898 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0327 23:07:38.394005 3898 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0327 23:07:42.158509 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:07:42.158532 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:07:42.158540 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 23:07:42.185941 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:07:42.185958 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 23:07:42.185964 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0327 23:07:47.672488 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=749 W0327 23:07:57.489661 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 23:08:02.645634 3898 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:08:02.646197 3898 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:08:02.662841 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=106 I0327 23:08:02.669737 3898 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:08:02.669758 3898 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:08:02.675297 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0327 23:08:02.675319 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0327 23:08:03.936510 3898 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:08:03.936618 3898 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 23:08:03.953615 3898 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 17.021015ms I0327 23:08:03.953648 3898 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 17.059063ms I0327 23:08:03.955183 3898 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 18.691356ms, libovsdb time 979.207µs I0327 23:08:04.301019 3898 cni.go:403] [tsf-keycloak/keycloak-0 1191a9a463a4db4fbeef0372c7075d208e5136add7c847721b78fdf69ab2795e network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 1191a9a463a4db4fbeef0372c7075d208e5136add7c847721b78fdf69ab2795e network default NAD default] I0327 23:08:04.309492 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 1191a9a463a4db4, network: default, NAD default, SandboxID: "1191a9a463a4db4fbeef0372c7075d208e5136add7c847721b78fdf69ab2795e", PCI device ID: , UID: "9ca9a396-3a53-4a43-bff5-5b9eb19de9f4", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0327 23:08:04.355286 3898 cni.go:424] [tsf-keycloak/keycloak-0 1191a9a463a4db4fbeef0372c7075d208e5136add7c847721b78fdf69ab2795e network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 1191a9a463a4db4fbeef0372c7075d208e5136add7c847721b78fdf69ab2795e network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:cd:14:2f:b6:67\",\"name\":\"1191a9a463a4db4\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d0c6551c-45e9-4bf0-8bde-e2313e094cd7\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:08:15.386124 3898 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 23:08:15.661522 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=84 I0327 23:08:16.743321 3898 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 W0327 23:08:24.417248 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:24.417268 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:24.419004 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 23:08:35.234668 3898 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-mz22v] creating logical port openshift-pipelines_tekton-results-watcher-6f946f97cf-mz22v for pod on switch ip-10-0-0-48.ec2.internal I0327 23:08:35.234790 3898 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-6f946f97cf-mz22v I0327 23:08:35.247229 3898 pod.go:62] [openshift-pipelines/tekton-results-watcher-6f946f97cf-mz22v] pod update took 12.461171ms I0327 23:08:35.247255 3898 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-mz22v] addLogicalPort annotation time took 12.49153ms I0327 23:08:35.248769 3898 pods.go:271] [openshift-pipelines/tekton-results-watcher-6f946f97cf-mz22v] addLogicalPort took 14.121854ms, libovsdb time 955.112µs I0327 23:08:35.595523 3898 cni.go:403] [openshift-pipelines/tekton-results-watcher-6f946f97cf-mz22v 63206782ab5dff49f38dd9b4dfae932a0e0a3efc5891d207c4151cb68fb2ed79 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-mz22v 63206782ab5dff49f38dd9b4dfae932a0e0a3efc5891d207c4151cb68fb2ed79 network default NAD default] I0327 23:08:35.604045 3898 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-6f946f97cf-mz22v, hostIfaceName: 63206782ab5dff4, network: default, NAD default, SandboxID: "63206782ab5dff49f38dd9b4dfae932a0e0a3efc5891d207c4151cb68fb2ed79", PCI device ID: , UID: "61449595-82ec-446c-bd68-5410f4970c9f", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0327 23:08:35.651576 3898 cni.go:424] [openshift-pipelines/tekton-results-watcher-6f946f97cf-mz22v 63206782ab5dff49f38dd9b4dfae932a0e0a3efc5891d207c4151cb68fb2ed79 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-mz22v 63206782ab5dff49f38dd9b4dfae932a0e0a3efc5891d207c4151cb68fb2ed79 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:03:d1:9d:45:c2\",\"name\":\"63206782ab5dff4\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2596b379-f8f6-4410-b053-17c03117fcf9\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:08:43.711316 3898 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 23:08:43.711335 3898 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 23:08:43.711346 3898 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 23:08:45.555824 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:45.555849 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:45.558386 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 23:08:46.668511 3898 cni.go:403] [tsf-keycloak/keycloak-0 1191a9a463a4db4fbeef0372c7075d208e5136add7c847721b78fdf69ab2795e network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 1191a9a463a4db4fbeef0372c7075d208e5136add7c847721b78fdf69ab2795e network default NAD default] I0327 23:08:46.741011 3898 cni.go:424] [tsf-keycloak/keycloak-0 1191a9a463a4db4fbeef0372c7075d208e5136add7c847721b78fdf69ab2795e network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 1191a9a463a4db4fbeef0372c7075d208e5136add7c847721b78fdf69ab2795e network default NAD default], result "{}", err I0327 23:08:46.792997 3898 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:08:46.793029 3898 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0327 23:08:46.794277 3898 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.130.0.41 I0327 23:08:46.794304 3898 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:08:47.556865 3898 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:08:47.556893 3898 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:08:47.562182 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 23:08:47.562211 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 23:08:47.597887 3898 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:08:47.597970 3898 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 23:08:47.614459 3898 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 16.513625ms I0327 23:08:47.614495 3898 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 16.553644ms I0327 23:08:47.615860 3898 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 17.986092ms, libovsdb time 989.583µs I0327 23:08:47.959894 3898 cni.go:403] [tsf-keycloak/keycloak-0 bfc4d24ae2c67f7cd51f2f6dab126814a5a951a38fd4664ff210bb020010bd3c network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 bfc4d24ae2c67f7cd51f2f6dab126814a5a951a38fd4664ff210bb020010bd3c network default NAD default] I0327 23:08:47.968332 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: bfc4d24ae2c67f7, network: default, NAD default, SandboxID: "bfc4d24ae2c67f7cd51f2f6dab126814a5a951a38fd4664ff210bb020010bd3c", PCI device ID: , UID: "f04204e7-9f85-49e5-8b95-5392ea9f96fb", MAC: 0a:58:0a:82:00:2b, IPs: [10.130.0.43/23] I0327 23:08:48.014252 3898 cni.go:424] [tsf-keycloak/keycloak-0 bfc4d24ae2c67f7cd51f2f6dab126814a5a951a38fd4664ff210bb020010bd3c network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 bfc4d24ae2c67f7cd51f2f6dab126814a5a951a38fd4664ff210bb020010bd3c network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:4a:83:de:bc:4c\",\"name\":\"bfc4d24ae2c67f7\"},{\"mac\":\"0a:58:0a:82:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eb02de6d-d3fe-4c10-bf8d-798acf98bcfe\"}],\"ips\":[{\"address\":\"10.130.0.43/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:08:49.577418 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:49.577578 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:08:49.577504 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 23:09:03.280299 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 23:09:03.281227 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:09:03.281248 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak I0327 23:09:04.145360 3898 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-gvqnc] creating logical port tsf-keycloak_tsf-iam-gvqnc for pod on switch ip-10-0-0-48.ec2.internal I0327 23:09:04.145529 3898 kube.go:256] Updating pod tsf-keycloak/tsf-iam-gvqnc I0327 23:09:04.158674 3898 pod.go:62] [tsf-keycloak/tsf-iam-gvqnc] pod update took 13.169879ms I0327 23:09:04.158698 3898 base_network_controller_pods.go:937] [default/tsf-keycloak/tsf-iam-gvqnc] addLogicalPort annotation time took 13.199148ms I0327 23:09:04.160085 3898 pods.go:271] [tsf-keycloak/tsf-iam-gvqnc] addLogicalPort took 14.752143ms, libovsdb time 858.466µs I0327 23:09:04.513803 3898 cni.go:403] [tsf-keycloak/tsf-iam-gvqnc b07baaaf763c284f16160050932be95f5dd6c82dca0638685e8d404164ce5d24 network default NAD default] ADD starting CNI request [tsf-keycloak/tsf-iam-gvqnc b07baaaf763c284f16160050932be95f5dd6c82dca0638685e8d404164ce5d24 network default NAD default] I0327 23:09:04.522445 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-gvqnc, hostIfaceName: b07baaaf763c284, network: default, NAD default, SandboxID: "b07baaaf763c284f16160050932be95f5dd6c82dca0638685e8d404164ce5d24", PCI device ID: , UID: "e560fcf8-8af7-40fa-869a-a4bf84e87ad4", MAC: 0a:58:0a:82:00:2c, IPs: [10.130.0.44/23] I0327 23:09:04.568296 3898 cni.go:424] [tsf-keycloak/tsf-iam-gvqnc b07baaaf763c284f16160050932be95f5dd6c82dca0638685e8d404164ce5d24 network default NAD default] ADD finished CNI request [tsf-keycloak/tsf-iam-gvqnc b07baaaf763c284f16160050932be95f5dd6c82dca0638685e8d404164ce5d24 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:51:66:f3:8b:d4\",\"name\":\"b07baaaf763c284\"},{\"mac\":\"0a:58:0a:82:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/50cfdcd4-69e4-46b6-bb6f-b9a1ee4c3258\"}],\"ips\":[{\"address\":\"10.130.0.44/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:09:05.481985 3898 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:05.482569 3898 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:22.779841 3898 cni.go:403] [tsf-keycloak/tsf-iam-gvqnc b07baaaf763c284f16160050932be95f5dd6c82dca0638685e8d404164ce5d24 network default NAD default] DEL starting CNI request [tsf-keycloak/tsf-iam-gvqnc b07baaaf763c284f16160050932be95f5dd6c82dca0638685e8d404164ce5d24 network default NAD default] I0327 23:09:22.852603 3898 cni.go:424] [tsf-keycloak/tsf-iam-gvqnc b07baaaf763c284f16160050932be95f5dd6c82dca0638685e8d404164ce5d24 network default NAD default] DEL finished CNI request [tsf-keycloak/tsf-iam-gvqnc b07baaaf763c284f16160050932be95f5dd6c82dca0638685e8d404164ce5d24 network default NAD default], result "{}", err I0327 23:09:22.903793 3898 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-gvqnc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:22.903828 3898 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-gvqnc I0327 23:09:22.905164 3898 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-gvqnc, ips: 10.130.0.44 I0327 23:09:22.905196 3898 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-gvqnc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:23.930748 3898 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-gvqnc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:09:23.930779 3898 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-gvqnc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:09:24.153753 3898 cni.go:403] [tsf-keycloak/keycloak-0 bfc4d24ae2c67f7cd51f2f6dab126814a5a951a38fd4664ff210bb020010bd3c network default NAD default] DEL starting CNI request [tsf-keycloak/keycloak-0 bfc4d24ae2c67f7cd51f2f6dab126814a5a951a38fd4664ff210bb020010bd3c network default NAD default] I0327 23:09:24.221792 3898 cni.go:424] [tsf-keycloak/keycloak-0 bfc4d24ae2c67f7cd51f2f6dab126814a5a951a38fd4664ff210bb020010bd3c network default NAD default] DEL finished CNI request [tsf-keycloak/keycloak-0 bfc4d24ae2c67f7cd51f2f6dab126814a5a951a38fd4664ff210bb020010bd3c network default NAD default], result "{}", err I0327 23:09:24.274164 3898 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:24.274197 3898 pods.go:176] Deleting pod: tsf-keycloak/keycloak-0 I0327 23:09:24.275430 3898 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/keycloak-0, ips: 10.130.0.43 I0327 23:09:24.275459 3898 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:09:24.784416 3898 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:09:24.784450 3898 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:09:24.790080 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 23:09:24.790111 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 23:09:24.820696 3898 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:09:24.820783 3898 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0327 23:09:24.835181 3898 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 14.424879ms I0327 23:09:24.835199 3898 base_network_controller_pods.go:937] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 14.445377ms I0327 23:09:24.836570 3898 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 15.891386ms, libovsdb time 930.262µs I0327 23:09:25.182103 3898 cni.go:403] [tsf-keycloak/keycloak-0 77ba74181779d7bd6a85466f0164f5682d52c8817079a381d699c0d423643ad2 network default NAD default] ADD starting CNI request [tsf-keycloak/keycloak-0 77ba74181779d7bd6a85466f0164f5682d52c8817079a381d699c0d423643ad2 network default NAD default] I0327 23:09:25.190827 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: 77ba74181779d7b, network: default, NAD default, SandboxID: "77ba74181779d7bd6a85466f0164f5682d52c8817079a381d699c0d423643ad2", PCI device ID: , UID: "b186bd77-86c9-4a5f-9620-2552fa0efb89", MAC: 0a:58:0a:82:00:2d, IPs: [10.130.0.45/23] I0327 23:09:25.237944 3898 cni.go:424] [tsf-keycloak/keycloak-0 77ba74181779d7bd6a85466f0164f5682d52c8817079a381d699c0d423643ad2 network default NAD default] ADD finished CNI request [tsf-keycloak/keycloak-0 77ba74181779d7bd6a85466f0164f5682d52c8817079a381d699c0d423643ad2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:68:0a:75:74:40\",\"name\":\"77ba74181779d7b\"},{\"mac\":\"0a:58:0a:82:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b3b1ca22-6af9-439f-af1f-4b29fb99f581\"}],\"ips\":[{\"address\":\"10.130.0.45/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:09:26.810807 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:09:26.810839 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:09:26.816204 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 23:09:28.956665 3898 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0327 23:09:29.013688 3898 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0327 23:09:29.013724 3898 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0327 23:09:31.370178 3898 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-48.ec2.internal I0327 23:09:31.370297 3898 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-48.ec2.internal: 136.731µs I0327 23:09:31.370306 3898 egressfirewall.go:628] Syncing node "ip-10-0-0-48.ec2.internal" for egress firewall I0327 23:09:31.396032 3898 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-gns7s] creating logical port kube-system_global-pull-secret-syncer-gns7s for pod on switch ip-10-0-0-48.ec2.internal I0327 23:09:31.396146 3898 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-gns7s I0327 23:09:31.414662 3898 pod.go:62] [kube-system/global-pull-secret-syncer-gns7s] pod update took 18.545163ms I0327 23:09:31.414690 3898 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-gns7s] addLogicalPort annotation time took 18.579104ms I0327 23:09:31.416110 3898 pods.go:271] [kube-system/global-pull-secret-syncer-gns7s] addLogicalPort took 20.101093ms, libovsdb time 1.002447ms I0327 23:09:31.773038 3898 cni.go:403] [kube-system/global-pull-secret-syncer-gns7s c550d4bc3d31ca71967f7b0277598910095ff5c8ceabb7cb5368130162c9e27f network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-gns7s c550d4bc3d31ca71967f7b0277598910095ff5c8ceabb7cb5368130162c9e27f network default NAD default] I0327 23:09:31.781347 3898 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-gns7s, hostIfaceName: c550d4bc3d31ca7, network: default, NAD default, SandboxID: "c550d4bc3d31ca71967f7b0277598910095ff5c8ceabb7cb5368130162c9e27f", PCI device ID: , UID: "04878096-56b5-4918-ba0a-0f37600c08d2", MAC: 0a:58:0a:82:00:2e, IPs: [10.130.0.46/23] I0327 23:09:31.830741 3898 cni.go:424] [kube-system/global-pull-secret-syncer-gns7s c550d4bc3d31ca71967f7b0277598910095ff5c8ceabb7cb5368130162c9e27f network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-gns7s c550d4bc3d31ca71967f7b0277598910095ff5c8ceabb7cb5368130162c9e27f network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:fb:81:4f:6c:96\",\"name\":\"c550d4bc3d31ca7\"},{\"mac\":\"0a:58:0a:82:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9d5342d5-fbab-48f9-a85e-27e501cf9e36\"}],\"ips\":[{\"address\":\"10.130.0.46/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:09:40.497487 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:09:40.497514 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0327 23:09:40.499878 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0327 23:09:59.769169 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0327 23:10:04.337369 3898 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-zcpwn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:04.337981 3898 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-zcpwn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:04.746482 3898 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0327 23:10:05.221005 3898 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-zcpwn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:05.221033 3898 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577550-zcpwn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:05.598687 3898 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:05.598717 3898 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:05.605954 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 23:10:05.605983 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 23:10:14.335436 3898 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:14.335992 3898 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:14.355322 3898 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:14.355343 3898 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:14.365506 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 23:10:14.365532 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 23:10:15.998063 3898 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-vvwth-lxlpn] creating logical port tsf-tas_segment-backup-installation-vvwth-lxlpn for pod on switch ip-10-0-0-48.ec2.internal I0327 23:10:15.998176 3898 kube.go:256] Updating pod tsf-tas/segment-backup-installation-vvwth-lxlpn I0327 23:10:16.012027 3898 pod.go:62] [tsf-tas/segment-backup-installation-vvwth-lxlpn] pod update took 13.883238ms I0327 23:10:16.012059 3898 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-vvwth-lxlpn] addLogicalPort annotation time took 13.91998ms I0327 23:10:16.017236 3898 pods.go:271] [tsf-tas/segment-backup-installation-vvwth-lxlpn] addLogicalPort took 19.195692ms, libovsdb time 4.6579ms I0327 23:10:16.367279 3898 cni.go:403] [tsf-tas/segment-backup-installation-vvwth-lxlpn 8c456d0307ba8f065c79139b18e40c8cb69362ebeede6241d1c690ec4a22784f network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-vvwth-lxlpn 8c456d0307ba8f065c79139b18e40c8cb69362ebeede6241d1c690ec4a22784f network default NAD default] I0327 23:10:16.376025 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-vvwth-lxlpn, hostIfaceName: 8c456d0307ba8f0, network: default, NAD default, SandboxID: "8c456d0307ba8f065c79139b18e40c8cb69362ebeede6241d1c690ec4a22784f", PCI device ID: , UID: "f0abbeef-16a1-41c7-8461-835c218b5e47", MAC: 0a:58:0a:82:00:2f, IPs: [10.130.0.47/23] I0327 23:10:16.425748 3898 cni.go:424] [tsf-tas/segment-backup-installation-vvwth-lxlpn 8c456d0307ba8f065c79139b18e40c8cb69362ebeede6241d1c690ec4a22784f network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-vvwth-lxlpn 8c456d0307ba8f065c79139b18e40c8cb69362ebeede6241d1c690ec4a22784f network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:5f:3d:84:0b:89\",\"name\":\"8c456d0307ba8f0\"},{\"mac\":\"0a:58:0a:82:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9570fed2-08d3-4d19-8c46-fd148f082c25\"}],\"ips\":[{\"address\":\"10.130.0.47/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:10:17.150017 3898 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-676d66b849-gb2bc] creating logical port tsf-tas_fulcio-server-676d66b849-gb2bc for pod on switch ip-10-0-0-48.ec2.internal I0327 23:10:17.150240 3898 kube.go:256] Updating pod tsf-tas/fulcio-server-676d66b849-gb2bc I0327 23:10:17.169875 3898 pod.go:62] [tsf-tas/fulcio-server-676d66b849-gb2bc] pod update took 19.669365ms I0327 23:10:17.169967 3898 base_network_controller_pods.go:937] [default/tsf-tas/fulcio-server-676d66b849-gb2bc] addLogicalPort annotation time took 19.766539ms I0327 23:10:17.171791 3898 pods.go:271] [tsf-tas/fulcio-server-676d66b849-gb2bc] addLogicalPort took 21.794893ms, libovsdb time 1.178793ms I0327 23:10:17.207873 3898 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-m9vwn-l2rxq] creating logical port tsf-tas_ctlog-createtree-job-m9vwn-l2rxq for pod on switch ip-10-0-0-48.ec2.internal I0327 23:10:17.208082 3898 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-m9vwn-l2rxq I0327 23:10:17.230180 3898 pod.go:62] [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq] pod update took 22.124017ms I0327 23:10:17.230210 3898 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-createtree-job-m9vwn-l2rxq] addLogicalPort annotation time took 22.160049ms I0327 23:10:17.231892 3898 pods.go:271] [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq] addLogicalPort took 24.037331ms, libovsdb time 1.107325ms I0327 23:10:17.419526 3898 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-5b795f9c7f-jpgzh] creating logical port tsf-tas_trillian-logsigner-5b795f9c7f-jpgzh for pod on switch ip-10-0-0-48.ec2.internal I0327 23:10:17.419624 3898 kube.go:256] Updating pod tsf-tas/trillian-logsigner-5b795f9c7f-jpgzh I0327 23:10:17.442158 3898 pod.go:62] [tsf-tas/trillian-logsigner-5b795f9c7f-jpgzh] pod update took 22.559741ms I0327 23:10:17.442256 3898 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logsigner-5b795f9c7f-jpgzh] addLogicalPort annotation time took 22.662241ms I0327 23:10:17.443899 3898 pods.go:271] [tsf-tas/trillian-logsigner-5b795f9c7f-jpgzh] addLogicalPort took 24.390234ms, libovsdb time 1.024458ms I0327 23:10:17.530356 3898 cni.go:403] [tsf-tas/fulcio-server-676d66b849-gb2bc cd780df12fe2f0854da8654df31f0d4f9f4e9b1da5dc61a51b8d46b559d8fa9a network default NAD default] ADD starting CNI request [tsf-tas/fulcio-server-676d66b849-gb2bc cd780df12fe2f0854da8654df31f0d4f9f4e9b1da5dc61a51b8d46b559d8fa9a network default NAD default] I0327 23:10:17.541235 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-676d66b849-gb2bc, hostIfaceName: cd780df12fe2f08, network: default, NAD default, SandboxID: "cd780df12fe2f0854da8654df31f0d4f9f4e9b1da5dc61a51b8d46b559d8fa9a", PCI device ID: , UID: "3782ce5e-a9d2-4517-84b3-cc2c6caedf0d", MAC: 0a:58:0a:82:00:30, IPs: [10.130.0.48/23] I0327 23:10:17.580330 3898 cni.go:403] [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq 67a164922d3cd1b8210dd642a37bf5b2ce27926125a9923a19f289171d6b4418 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq 67a164922d3cd1b8210dd642a37bf5b2ce27926125a9923a19f289171d6b4418 network default NAD default] I0327 23:10:17.593390 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-m9vwn-l2rxq, hostIfaceName: 67a164922d3cd1b, network: default, NAD default, SandboxID: "67a164922d3cd1b8210dd642a37bf5b2ce27926125a9923a19f289171d6b4418", PCI device ID: , UID: "bff6bbb5-5c83-40ca-bca4-c64ab5fafc7b", MAC: 0a:58:0a:82:00:31, IPs: [10.130.0.49/23] I0327 23:10:17.602747 3898 cni.go:424] [tsf-tas/fulcio-server-676d66b849-gb2bc cd780df12fe2f0854da8654df31f0d4f9f4e9b1da5dc61a51b8d46b559d8fa9a network default NAD default] ADD finished CNI request [tsf-tas/fulcio-server-676d66b849-gb2bc cd780df12fe2f0854da8654df31f0d4f9f4e9b1da5dc61a51b8d46b559d8fa9a network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:b8:22:2d:65:d5\",\"name\":\"cd780df12fe2f08\"},{\"mac\":\"0a:58:0a:82:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ad77e4ca-3525-41e2-93d9-06be90a1fc23\"}],\"ips\":[{\"address\":\"10.130.0.48/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:10:17.654670 3898 cni.go:424] [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq 67a164922d3cd1b8210dd642a37bf5b2ce27926125a9923a19f289171d6b4418 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq 67a164922d3cd1b8210dd642a37bf5b2ce27926125a9923a19f289171d6b4418 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:62:a6:b7:57:c4\",\"name\":\"67a164922d3cd1b\"},{\"mac\":\"0a:58:0a:82:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/892f3dde-b0d2-41c2-ac7d-95b1ea44f176\"}],\"ips\":[{\"address\":\"10.130.0.49/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:10:17.795171 3898 cni.go:403] [tsf-tas/trillian-logsigner-5b795f9c7f-jpgzh e31f077842a6093e0c953de49aec4eff55b744973c4533e53d45ec0ae7adff36 network default NAD default] ADD starting CNI request [tsf-tas/trillian-logsigner-5b795f9c7f-jpgzh e31f077842a6093e0c953de49aec4eff55b744973c4533e53d45ec0ae7adff36 network default NAD default] I0327 23:10:17.804436 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-5b795f9c7f-jpgzh, hostIfaceName: e31f077842a6093, network: default, NAD default, SandboxID: "e31f077842a6093e0c953de49aec4eff55b744973c4533e53d45ec0ae7adff36", PCI device ID: , UID: "99793c2c-c1e2-4474-99e1-ebfc15f4a01a", MAC: 0a:58:0a:82:00:32, IPs: [10.130.0.50/23] I0327 23:10:17.856106 3898 cni.go:424] [tsf-tas/trillian-logsigner-5b795f9c7f-jpgzh e31f077842a6093e0c953de49aec4eff55b744973c4533e53d45ec0ae7adff36 network default NAD default] ADD finished CNI request [tsf-tas/trillian-logsigner-5b795f9c7f-jpgzh e31f077842a6093e0c953de49aec4eff55b744973c4533e53d45ec0ae7adff36 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:45:bb:cb:ca:8a\",\"name\":\"e31f077842a6093\"},{\"mac\":\"0a:58:0a:82:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/29e97560-f961-474c-93b0-b753d8fea685\"}],\"ips\":[{\"address\":\"10.130.0.50/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:10:22.112872 3898 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0327 23:10:22.112896 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0327 23:10:23.129408 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server W0327 23:10:23.129436 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0327 23:10:25.316064 3898 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0327 23:10:28.154088 3898 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0327 23:10:32.351268 3898 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0327 23:10:41.231456 3898 cni.go:403] [tsf-tas/segment-backup-installation-vvwth-lxlpn 8c456d0307ba8f065c79139b18e40c8cb69362ebeede6241d1c690ec4a22784f network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-vvwth-lxlpn 8c456d0307ba8f065c79139b18e40c8cb69362ebeede6241d1c690ec4a22784f network default NAD default] I0327 23:10:41.324325 3898 cni.go:424] [tsf-tas/segment-backup-installation-vvwth-lxlpn 8c456d0307ba8f065c79139b18e40c8cb69362ebeede6241d1c690ec4a22784f network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-vvwth-lxlpn 8c456d0307ba8f065c79139b18e40c8cb69362ebeede6241d1c690ec4a22784f network default NAD default], result "{}", err I0327 23:10:41.373641 3898 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-vvwth-lxlpn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:41.373675 3898 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-vvwth-lxlpn I0327 23:10:41.375136 3898 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-vvwth-lxlpn, ips: 10.130.0.47 I0327 23:10:41.375160 3898 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-vvwth-lxlpn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:10:42.238859 3898 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-vvwth-lxlpn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:42.238888 3898 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-vvwth-lxlpn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:10:45.780703 3898 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 23:10:57.387309 3898 namespace.go:142] [openshift-pipelines] updating namespace W0327 23:11:01.291724 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0327 23:11:01.663518 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0327 23:11:04.594374 3898 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:04.595002 3898 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:04.614881 3898 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:04.614899 3898 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:04.620645 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0327 23:11:04.620668 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0327 23:11:12.652608 3898 obj_retry.go:464] Detected object tsf/patch-tekton-config-l4pwv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:12.653177 3898 obj_retry.go:464] Detected object tsf/patch-tekton-config-l4pwv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:13.684022 3898 obj_retry.go:458] Detected object tsf/patch-tekton-config-l4pwv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:13.684046 3898 obj_retry.go:458] Detected object tsf/patch-tekton-config-l4pwv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:15.667753 3898 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-wpwm6-x5grc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:15.668314 3898 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-wpwm6-x5grc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:16.552496 3898 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-wpwm6-x5grc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:16.552523 3898 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-wpwm6-x5grc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:17.429817 3898 cni.go:403] [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq 67a164922d3cd1b8210dd642a37bf5b2ce27926125a9923a19f289171d6b4418 network default NAD default] DEL starting CNI request [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq 67a164922d3cd1b8210dd642a37bf5b2ce27926125a9923a19f289171d6b4418 network default NAD default] I0327 23:11:17.502612 3898 cni.go:424] [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq 67a164922d3cd1b8210dd642a37bf5b2ce27926125a9923a19f289171d6b4418 network default NAD default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-m9vwn-l2rxq 67a164922d3cd1b8210dd642a37bf5b2ce27926125a9923a19f289171d6b4418 network default NAD default], result "{}", err I0327 23:11:17.551130 3898 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-m9vwn-l2rxq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:17.551178 3898 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-m9vwn-l2rxq I0327 23:11:17.552715 3898 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-m9vwn-l2rxq, ips: 10.130.0.49 I0327 23:11:17.552752 3898 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-m9vwn-l2rxq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:18.441166 3898 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-m9vwn-l2rxq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.441199 3898 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-m9vwn-l2rxq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:18.503127 3898 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-564c4495d-524rn] creating logical port tsf-tas_rekor-redis-564c4495d-524rn for pod on switch ip-10-0-0-48.ec2.internal I0327 23:11:18.503236 3898 kube.go:256] Updating pod tsf-tas/rekor-redis-564c4495d-524rn I0327 23:11:18.515767 3898 pod.go:62] [tsf-tas/rekor-redis-564c4495d-524rn] pod update took 12.556796ms I0327 23:11:18.515862 3898 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-564c4495d-524rn] addLogicalPort annotation time took 12.655441ms I0327 23:11:18.517215 3898 pods.go:271] [tsf-tas/rekor-redis-564c4495d-524rn] addLogicalPort took 14.113481ms, libovsdb time 838.369µs I0327 23:11:19.464704 3898 cni.go:403] [tsf-tas/rekor-redis-564c4495d-524rn c44fcb6950b52be04d0437c260bfc03b17445a91adf438d19d7190bd5afb2e47 network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-564c4495d-524rn c44fcb6950b52be04d0437c260bfc03b17445a91adf438d19d7190bd5afb2e47 network default NAD default] I0327 23:11:19.473371 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-564c4495d-524rn, hostIfaceName: c44fcb6950b52be, network: default, NAD default, SandboxID: "c44fcb6950b52be04d0437c260bfc03b17445a91adf438d19d7190bd5afb2e47", PCI device ID: , UID: "0a4f2a61-cf21-4d28-af7f-c386b4d1598d", MAC: 0a:58:0a:82:00:33, IPs: [10.130.0.51/23] I0327 23:11:19.521546 3898 cni.go:424] [tsf-tas/rekor-redis-564c4495d-524rn c44fcb6950b52be04d0437c260bfc03b17445a91adf438d19d7190bd5afb2e47 network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-564c4495d-524rn c44fcb6950b52be04d0437c260bfc03b17445a91adf438d19d7190bd5afb2e47 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:80:62:57:9a:cf\",\"name\":\"c44fcb6950b52be\"},{\"mac\":\"0a:58:0a:82:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0ccb59ba-5194-4dca-afee-8551b3c776c0\"}],\"ips\":[{\"address\":\"10.130.0.51/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:11:21.459659 3898 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0327 23:11:22.577801 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0327 23:11:22.755838 3898 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-rn4wv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:22.756392 3898 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-rn4wv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:23.773560 3898 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-rn4wv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:23.773629 3898 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-rn4wv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:23.835221 3898 obj_retry.go:458] Detected object tsf/patch-tekton-config-l4pwv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:23.835251 3898 obj_retry.go:458] Detected object tsf/patch-tekton-config-l4pwv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:23.839246 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-l4pwv I0327 23:11:23.839262 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-l4pwv I0327 23:11:23.966953 3898 base_network_controller_pods.go:487] [default/tsf/test-tsf-pipelines] creating logical port tsf_test-tsf-pipelines for pod on switch ip-10-0-0-48.ec2.internal I0327 23:11:23.967088 3898 kube.go:256] Updating pod tsf/test-tsf-pipelines I0327 23:11:23.978517 3898 pod.go:62] [tsf/test-tsf-pipelines] pod update took 11.454307ms I0327 23:11:23.978539 3898 base_network_controller_pods.go:937] [default/tsf/test-tsf-pipelines] addLogicalPort annotation time took 11.485429ms I0327 23:11:23.980116 3898 pods.go:271] [tsf/test-tsf-pipelines] addLogicalPort took 13.179855ms, libovsdb time 1.080334ms I0327 23:11:24.329281 3898 cni.go:403] [tsf/test-tsf-pipelines 090cddd34b187cc2939d51b4f5e93ad2c967f38dbda95b5a83f88c6118865bad network default NAD default] ADD starting CNI request [tsf/test-tsf-pipelines 090cddd34b187cc2939d51b4f5e93ad2c967f38dbda95b5a83f88c6118865bad network default NAD default] I0327 23:11:24.337869 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-pipelines, hostIfaceName: 090cddd34b187cc, network: default, NAD default, SandboxID: "090cddd34b187cc2939d51b4f5e93ad2c967f38dbda95b5a83f88c6118865bad", PCI device ID: , UID: "dfea8594-bc74-4707-8348-a888f7c0ffdb", MAC: 0a:58:0a:82:00:34, IPs: [10.130.0.52/23] I0327 23:11:24.387907 3898 cni.go:424] [tsf/test-tsf-pipelines 090cddd34b187cc2939d51b4f5e93ad2c967f38dbda95b5a83f88c6118865bad network default NAD default] ADD finished CNI request [tsf/test-tsf-pipelines 090cddd34b187cc2939d51b4f5e93ad2c967f38dbda95b5a83f88c6118865bad network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:42:98:d3:33:c6\",\"name\":\"090cddd34b187cc\"},{\"mac\":\"0a:58:0a:82:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6c3ef8b7-65d1-4d76-aab1-0976224e35a1\"}],\"ips\":[{\"address\":\"10.130.0.52/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:11:26.599233 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0327 23:11:33.496300 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0327 23:11:34.542920 3898 cni.go:403] [tsf/test-tsf-pipelines 090cddd34b187cc2939d51b4f5e93ad2c967f38dbda95b5a83f88c6118865bad network default NAD default] DEL starting CNI request [tsf/test-tsf-pipelines 090cddd34b187cc2939d51b4f5e93ad2c967f38dbda95b5a83f88c6118865bad network default NAD default] I0327 23:11:34.625743 3898 cni.go:424] [tsf/test-tsf-pipelines 090cddd34b187cc2939d51b4f5e93ad2c967f38dbda95b5a83f88c6118865bad network default NAD default] DEL finished CNI request [tsf/test-tsf-pipelines 090cddd34b187cc2939d51b4f5e93ad2c967f38dbda95b5a83f88c6118865bad network default NAD default], result "{}", err W0327 23:11:34.660381 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0327 23:11:34.678345 3898 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:34.678381 3898 pods.go:176] Deleting pod: tsf/test-tsf-pipelines I0327 23:11:34.679733 3898 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-pipelines, ips: 10.130.0.52 I0327 23:11:34.679757 3898 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:34.703282 3898 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:34.703306 3898 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:34.709750 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0327 23:11:34.709779 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0327 23:11:38.807222 3898 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:38.807814 3898 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:38.831772 3898 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:38.831805 3898 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:38.831922 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0327 23:11:38.831946 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0327 23:11:40.142557 3898 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-47htb] creating logical port tsf-tpa_create-db-47htb for pod on switch ip-10-0-0-48.ec2.internal I0327 23:11:40.142697 3898 kube.go:256] Updating pod tsf-tpa/create-db-47htb I0327 23:11:40.157253 3898 pod.go:62] [tsf-tpa/create-db-47htb] pod update took 14.577475ms I0327 23:11:40.157281 3898 base_network_controller_pods.go:937] [default/tsf-tpa/create-db-47htb] addLogicalPort annotation time took 14.610094ms I0327 23:11:40.158615 3898 pods.go:271] [tsf-tpa/create-db-47htb] addLogicalPort took 16.07507ms, libovsdb time 889.801µs I0327 23:11:40.503716 3898 cni.go:403] [tsf-tpa/create-db-47htb 75c234cd135a9c60af965a3f1f05daee05cd2edc3484107d4ababd2030507a4a network default NAD default] ADD starting CNI request [tsf-tpa/create-db-47htb 75c234cd135a9c60af965a3f1f05daee05cd2edc3484107d4ababd2030507a4a network default NAD default] I0327 23:11:40.512373 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-db-47htb, hostIfaceName: 75c234cd135a9c6, network: default, NAD default, SandboxID: "75c234cd135a9c60af965a3f1f05daee05cd2edc3484107d4ababd2030507a4a", PCI device ID: , UID: "826e2d39-40dd-406a-8a86-078a44f00557", MAC: 0a:58:0a:82:00:35, IPs: [10.130.0.53/23] I0327 23:11:40.561186 3898 cni.go:424] [tsf-tpa/create-db-47htb 75c234cd135a9c60af965a3f1f05daee05cd2edc3484107d4ababd2030507a4a network default NAD default] ADD finished CNI request [tsf-tpa/create-db-47htb 75c234cd135a9c60af965a3f1f05daee05cd2edc3484107d4ababd2030507a4a network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:3a:1f:3f:63:36\",\"name\":\"75c234cd135a9c6\"},{\"mac\":\"0a:58:0a:82:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c61f327c-9569-4dd1-962b-5ea38c693586\"}],\"ips\":[{\"address\":\"10.130.0.53/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:11:44.604853 3898 cni.go:403] [tsf-tpa/create-db-47htb 75c234cd135a9c60af965a3f1f05daee05cd2edc3484107d4ababd2030507a4a network default NAD default] DEL starting CNI request [tsf-tpa/create-db-47htb 75c234cd135a9c60af965a3f1f05daee05cd2edc3484107d4ababd2030507a4a network default NAD default] I0327 23:11:44.695335 3898 cni.go:424] [tsf-tpa/create-db-47htb 75c234cd135a9c60af965a3f1f05daee05cd2edc3484107d4ababd2030507a4a network default NAD default] DEL finished CNI request [tsf-tpa/create-db-47htb 75c234cd135a9c60af965a3f1f05daee05cd2edc3484107d4ababd2030507a4a network default NAD default], result "{}", err I0327 23:11:44.746444 3898 obj_retry.go:464] Detected object tsf-tpa/create-db-47htb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:44.746501 3898 pods.go:176] Deleting pod: tsf-tpa/create-db-47htb I0327 23:11:44.747850 3898 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-47htb, ips: 10.130.0.53 I0327 23:11:44.747881 3898 obj_retry.go:464] Detected object tsf-tpa/create-db-47htb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:45.776102 3898 obj_retry.go:458] Detected object tsf-tpa/create-db-47htb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:45.776139 3898 obj_retry.go:458] Detected object tsf-tpa/create-db-47htb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:45.871906 3898 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-v94m6] creating logical port tsf-tpa_migrate-db-v94m6 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:11:45.872037 3898 kube.go:256] Updating pod tsf-tpa/migrate-db-v94m6 I0327 23:11:45.884272 3898 pod.go:62] [tsf-tpa/migrate-db-v94m6] pod update took 12.253706ms I0327 23:11:45.884301 3898 base_network_controller_pods.go:937] [default/tsf-tpa/migrate-db-v94m6] addLogicalPort annotation time took 12.290116ms I0327 23:11:45.885883 3898 pods.go:271] [tsf-tpa/migrate-db-v94m6] addLogicalPort took 13.994826ms, libovsdb time 1.106757ms I0327 23:11:46.233280 3898 cni.go:403] [tsf-tpa/migrate-db-v94m6 3f648c060f0f137bc5785e05a0e17eee2d24ee5f93abc27ea1e6399349e41c77 network default NAD default] ADD starting CNI request [tsf-tpa/migrate-db-v94m6 3f648c060f0f137bc5785e05a0e17eee2d24ee5f93abc27ea1e6399349e41c77 network default NAD default] I0327 23:11:46.242338 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-v94m6, hostIfaceName: 3f648c060f0f137, network: default, NAD default, SandboxID: "3f648c060f0f137bc5785e05a0e17eee2d24ee5f93abc27ea1e6399349e41c77", PCI device ID: , UID: "1b328d61-e1c2-4b81-846a-ca4292e99d13", MAC: 0a:58:0a:82:00:36, IPs: [10.130.0.54/23] I0327 23:11:46.292314 3898 cni.go:424] [tsf-tpa/migrate-db-v94m6 3f648c060f0f137bc5785e05a0e17eee2d24ee5f93abc27ea1e6399349e41c77 network default NAD default] ADD finished CNI request [tsf-tpa/migrate-db-v94m6 3f648c060f0f137bc5785e05a0e17eee2d24ee5f93abc27ea1e6399349e41c77 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:bf:0a:de:0e:6a\",\"name\":\"3f648c060f0f137\"},{\"mac\":\"0a:58:0a:82:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f5ae26f-210e-4433-b68a-d575c09ea58a\"}],\"ips\":[{\"address\":\"10.130.0.54/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:11:48.630180 3898 cni.go:403] [tsf-tpa/migrate-db-v94m6 3f648c060f0f137bc5785e05a0e17eee2d24ee5f93abc27ea1e6399349e41c77 network default NAD default] DEL starting CNI request [tsf-tpa/migrate-db-v94m6 3f648c060f0f137bc5785e05a0e17eee2d24ee5f93abc27ea1e6399349e41c77 network default NAD default] I0327 23:11:48.717119 3898 cni.go:424] [tsf-tpa/migrate-db-v94m6 3f648c060f0f137bc5785e05a0e17eee2d24ee5f93abc27ea1e6399349e41c77 network default NAD default] DEL finished CNI request [tsf-tpa/migrate-db-v94m6 3f648c060f0f137bc5785e05a0e17eee2d24ee5f93abc27ea1e6399349e41c77 network default NAD default], result "{}", err I0327 23:11:48.766235 3898 obj_retry.go:464] Detected object tsf-tpa/migrate-db-v94m6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:48.766278 3898 pods.go:176] Deleting pod: tsf-tpa/migrate-db-v94m6 I0327 23:11:48.767887 3898 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-v94m6, ips: 10.130.0.54 I0327 23:11:48.767925 3898 obj_retry.go:464] Detected object tsf-tpa/migrate-db-v94m6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:48.900126 3898 obj_retry.go:458] Detected object tsf-tpa/migrate-db-v94m6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:48.900162 3898 obj_retry.go:458] Detected object tsf-tpa/migrate-db-v94m6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:49.066854 3898 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-tgp5h] creating logical port tsf-tpa_create-importers-tgp5h for pod on switch ip-10-0-0-48.ec2.internal I0327 23:11:49.066959 3898 kube.go:256] Updating pod tsf-tpa/create-importers-tgp5h I0327 23:11:49.079840 3898 pod.go:62] [tsf-tpa/create-importers-tgp5h] pod update took 12.908524ms I0327 23:11:49.079871 3898 base_network_controller_pods.go:937] [default/tsf-tpa/create-importers-tgp5h] addLogicalPort annotation time took 12.947784ms I0327 23:11:49.081278 3898 pods.go:271] [tsf-tpa/create-importers-tgp5h] addLogicalPort took 14.439278ms, libovsdb time 841.984µs I0327 23:11:49.428506 3898 cni.go:403] [tsf-tpa/create-importers-tgp5h 5314baa59a92822109c2b6a33c3fec93a28746691f81b7191436c308ade7bcf3 network default NAD default] ADD starting CNI request [tsf-tpa/create-importers-tgp5h 5314baa59a92822109c2b6a33c3fec93a28746691f81b7191436c308ade7bcf3 network default NAD default] I0327 23:11:49.437823 3898 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-tgp5h, hostIfaceName: 5314baa59a92822, network: default, NAD default, SandboxID: "5314baa59a92822109c2b6a33c3fec93a28746691f81b7191436c308ade7bcf3", PCI device ID: , UID: "f26c60d6-be83-445a-bf1d-d9a2ead59b93", MAC: 0a:58:0a:82:00:37, IPs: [10.130.0.55/23] I0327 23:11:49.694023 3898 cni.go:424] [tsf-tpa/create-importers-tgp5h 5314baa59a92822109c2b6a33c3fec93a28746691f81b7191436c308ade7bcf3 network default NAD default] ADD finished CNI request [tsf-tpa/create-importers-tgp5h 5314baa59a92822109c2b6a33c3fec93a28746691f81b7191436c308ade7bcf3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:9e:a9:66:88:df\",\"name\":\"5314baa59a92822\"},{\"mac\":\"0a:58:0a:82:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/511d7410-8273-4c13-931f-7bbf3307dfb0\"}],\"ips\":[{\"address\":\"10.130.0.55/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:11:51.649995 3898 cni.go:403] [tsf-tpa/create-importers-tgp5h 5314baa59a92822109c2b6a33c3fec93a28746691f81b7191436c308ade7bcf3 network default NAD default] DEL starting CNI request [tsf-tpa/create-importers-tgp5h 5314baa59a92822109c2b6a33c3fec93a28746691f81b7191436c308ade7bcf3 network default NAD default] I0327 23:11:51.733780 3898 cni.go:424] [tsf-tpa/create-importers-tgp5h 5314baa59a92822109c2b6a33c3fec93a28746691f81b7191436c308ade7bcf3 network default NAD default] DEL finished CNI request [tsf-tpa/create-importers-tgp5h 5314baa59a92822109c2b6a33c3fec93a28746691f81b7191436c308ade7bcf3 network default NAD default], result "{}", err I0327 23:11:51.785621 3898 obj_retry.go:464] Detected object tsf-tpa/create-importers-tgp5h of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:51.785653 3898 pods.go:176] Deleting pod: tsf-tpa/create-importers-tgp5h I0327 23:11:51.786960 3898 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-tgp5h, ips: 10.130.0.55 I0327 23:11:51.786991 3898 obj_retry.go:464] Detected object tsf-tpa/create-importers-tgp5h of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:52.805886 3898 obj_retry.go:458] Detected object tsf-tpa/create-importers-tgp5h of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:52.805913 3898 obj_retry.go:458] Detected object tsf-tpa/create-importers-tgp5h of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:57.918564 3898 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-qkrpw-29dcv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:57.919193 3898 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-qkrpw-29dcv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:11:58.814717 3898 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-qkrpw-29dcv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:11:58.814741 3898 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-qkrpw-29dcv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 23:12:05.853026 3898 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0327 23:12:13.895421 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0327 23:12:29.774029 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0327 23:12:34.115520 3898 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:34.116111 3898 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:34.136499 3898 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:34.136522 3898 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:34.139781 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0327 23:12:34.139803 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0327 23:12:35.101656 3898 namespace.go:98] [integration-service] adding namespace I0327 23:12:35.102589 3898 namespace.go:102] [integration-service] adding namespace took 910.992µs I0327 23:12:35.102956 3898 namespace.go:98] [image-controller] adding namespace I0327 23:12:35.103607 3898 namespace.go:102] [image-controller] adding namespace took 633.917µs I0327 23:12:35.110162 3898 namespace.go:98] [build-service] adding namespace I0327 23:12:35.110819 3898 namespace.go:102] [build-service] adding namespace took 642.597µs I0327 23:12:35.135889 3898 namespace.go:142] [integration-service] updating namespace I0327 23:12:35.157362 3898 namespace.go:142] [integration-service] updating namespace I0327 23:12:35.174393 3898 namespace.go:142] [image-controller] updating namespace I0327 23:12:35.220183 3898 namespace.go:142] [image-controller] updating namespace I0327 23:12:35.232918 3898 namespace.go:142] [build-service] updating namespace I0327 23:12:35.255310 3898 namespace.go:142] [build-service] updating namespace I0327 23:12:35.352413 3898 namespace.go:142] [build-service] updating namespace I0327 23:12:35.367964 3898 namespace.go:142] [integration-service] updating namespace I0327 23:12:35.392869 3898 namespace.go:98] [release-service] adding namespace I0327 23:12:35.393842 3898 namespace.go:102] [release-service] adding namespace took 948.727µs I0327 23:12:35.419906 3898 namespace.go:142] [release-service] updating namespace I0327 23:12:35.420832 3898 namespace.go:142] [konflux-ui] updating namespace I0327 23:12:35.476285 3898 namespace.go:142] [release-service] updating namespace I0327 23:12:35.504141 3898 base_network_controller_pods.go:487] [default/konflux-ui/tsf-konflux-public-key-vpgth] creating logical port konflux-ui_tsf-konflux-public-key-vpgth for pod on switch ip-10-0-0-48.ec2.internal I0327 23:12:35.504227 3898 kube.go:256] Updating pod konflux-ui/tsf-konflux-public-key-vpgth I0327 23:12:35.640697 3898 kube.go:256] Updating pod konflux-ui/tsf-konflux-public-key-vpgth I0327 23:12:35.646906 3898 namespace.go:98] [konflux-info] adding namespace I0327 23:12:35.647744 3898 namespace.go:102] [konflux-info] adding namespace took 820.384µs I0327 23:12:35.723431 3898 base_network_controller_pods.go:937] [default/konflux-ui/tsf-konflux-public-key-vpgth] addLogicalPort annotation time took 219.226887ms I0327 23:12:35.723457 3898 base_network_controller_pods.go:828] Released IPs: 10.130.0.56 for node: ip-10-0-0-48.ec2.internal I0327 23:12:35.723489 3898 pods.go:271] [konflux-ui/tsf-konflux-public-key-vpgth] addLogicalPort took 219.344464ms, libovsdb time 0s E0327 23:12:35.723510 3898 obj_retry.go:682] Failed to update *v1.Pod, old=konflux-ui/tsf-konflux-public-key-vpgth, new=konflux-ui/tsf-konflux-public-key-vpgth, error: addLogicalPort failed for konflux-ui/tsf-konflux-public-key-vpgth: failed to update pod konflux-ui/tsf-konflux-public-key-vpgth: Operation cannot be fulfilled on pods "tsf-konflux-public-key-vpgth": the object has been modified; please apply your changes to the latest version and try again I0327 23:12:35.723597 3898 base_network_controller_pods.go:487] [default/konflux-ui/tsf-konflux-public-key-vpgth] creating logical port konflux-ui_tsf-konflux-public-key-vpgth for pod on switch ip-10-0-0-48.ec2.internal I0327 23:12:35.723650 3898 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"konflux-ui", Name:"tsf-konflux-public-key-vpgth", UID:"d8198268-ff16-4a0a-b529-2454ee6fa1d5", APIVersion:"v1", ResourceVersion:"26407", FieldPath:""}): type: 'Warning' reason: 'ErrorUpdatingResource' addLogicalPort failed for konflux-ui/tsf-konflux-public-key-vpgth: failed to update pod konflux-ui/tsf-konflux-public-key-vpgth: Operation cannot be fulfilled on pods "tsf-konflux-public-key-vpgth": the object has been modified; please apply your changes to the latest version and try again I0327 23:12:35.723682 3898 kube.go:256] Updating pod konflux-ui/tsf-konflux-public-key-vpgth I0327 23:12:35.785418 3898 namespace.go:142] [konflux-info] updating namespace I0327 23:12:35.816303 3898 pod.go:62] [konflux-ui/tsf-konflux-public-key-vpgth] pod update took 92.643049ms I0327 23:12:35.816330 3898 base_network_controller_pods.go:937] [default/konflux-ui/tsf-konflux-public-key-vpgth] addLogicalPort annotation time took 92.678009ms I0327 23:12:35.817645 3898 pods.go:271] [konflux-ui/tsf-konflux-public-key-vpgth] addLogicalPort took 94.054829ms, libovsdb time 878.591µs I0327 23:12:35.824779 3898 namespace.go:98] [namespace-lister] adding namespace I0327 23:12:35.825580 3898 namespace.go:102] [namespace-lister] adding namespace took 777.909µs I0327 23:12:35.869877 3898 cni.go:403] [konflux-ui/tsf-konflux-public-key-vpgth 7a7edf28a01186813a56dfd22019001d00e6fc4a9c4172f3f1ed756bb814d1b6 network default NAD default] ADD starting CNI request [konflux-ui/tsf-konflux-public-key-vpgth 7a7edf28a01186813a56dfd22019001d00e6fc4a9c4172f3f1ed756bb814d1b6 network default NAD default] I0327 23:12:35.879560 3898 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: tsf-konflux-public-key-vpgth, hostIfaceName: 7a7edf28a011868, network: default, NAD default, SandboxID: "7a7edf28a01186813a56dfd22019001d00e6fc4a9c4172f3f1ed756bb814d1b6", PCI device ID: , UID: "d8198268-ff16-4a0a-b529-2454ee6fa1d5", MAC: 0a:58:0a:82:00:39, IPs: [10.130.0.57/23] I0327 23:12:35.917640 3898 namespace.go:142] [konflux-info] updating namespace I0327 23:12:35.930919 3898 cni.go:424] [konflux-ui/tsf-konflux-public-key-vpgth 7a7edf28a01186813a56dfd22019001d00e6fc4a9c4172f3f1ed756bb814d1b6 network default NAD default] ADD finished CNI request [konflux-ui/tsf-konflux-public-key-vpgth 7a7edf28a01186813a56dfd22019001d00e6fc4a9c4172f3f1ed756bb814d1b6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:99:c5:9b:0e:a0\",\"name\":\"7a7edf28a011868\"},{\"mac\":\"0a:58:0a:82:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ca4075e5-004c-4498-85ef-3deb50fcbc20\"}],\"ips\":[{\"address\":\"10.130.0.57/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:12:35.965210 3898 namespace.go:98] [enterprise-contract-service] adding namespace I0327 23:12:35.966107 3898 namespace.go:102] [enterprise-contract-service] adding namespace took 872.455µs I0327 23:12:36.074827 3898 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:36.074919 3898 namespace.go:142] [image-controller] updating namespace I0327 23:12:36.251205 3898 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:36.515155 3898 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:36.617988 3898 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:36.751454 3898 namespace.go:98] [default-tenant] adding namespace I0327 23:12:36.752406 3898 namespace.go:102] [default-tenant] adding namespace took 908.495µs I0327 23:12:36.921207 3898 namespace.go:98] [segment-bridge] adding namespace I0327 23:12:36.922201 3898 namespace.go:102] [segment-bridge] adding namespace took 930.128µs I0327 23:12:36.980848 3898 namespace.go:142] [default-tenant] updating namespace I0327 23:12:37.056137 3898 namespace.go:142] [default-tenant] updating namespace I0327 23:12:37.138504 3898 namespace.go:142] [segment-bridge] updating namespace I0327 23:12:37.189024 3898 namespace.go:142] [segment-bridge] updating namespace I0327 23:12:37.279966 3898 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0327 23:12:37.280646 3898 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0327 23:12:37.280663 3898 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0327 23:12:37.283402 3898 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0327 23:12:37.372288 3898 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0327 23:12:37.372318 3898 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0327 23:12:37.373200 3898 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0327 23:12:37.767025 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0327 23:12:37.771804 3898 namespace.go:142] [build-service] updating namespace I0327 23:12:38.178846 3898 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:38.239098 3898 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8] creating logical port image-controller_image-controller-controller-manager-5f8cd64d55-jr9z8 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:12:38.239216 3898 kube.go:256] Updating pod image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 I0327 23:12:38.262066 3898 pod.go:62] [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8] pod update took 22.879361ms I0327 23:12:38.262095 3898 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8] addLogicalPort annotation time took 22.914196ms I0327 23:12:38.263431 3898 pods.go:271] [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8] addLogicalPort took 24.3539ms, libovsdb time 897.842µs I0327 23:12:38.453983 3898 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7df75dd4df-lv9t6] creating logical port build-service_build-service-controller-manager-7df75dd4df-lv9t6 for pod on switch ip-10-0-0-48.ec2.internal I0327 23:12:38.454114 3898 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-lv9t6 I0327 23:12:38.473928 3898 pod.go:62] [build-service/build-service-controller-manager-7df75dd4df-lv9t6] pod update took 19.83552ms I0327 23:12:38.473952 3898 base_network_controller_pods.go:937] [default/build-service/build-service-controller-manager-7df75dd4df-lv9t6] addLogicalPort annotation time took 19.867425ms I0327 23:12:38.475140 3898 pods.go:271] [build-service/build-service-controller-manager-7df75dd4df-lv9t6] addLogicalPort took 21.170376ms, libovsdb time 781.361µs I0327 23:12:38.537784 3898 namespace.go:142] [image-controller] updating namespace I0327 23:12:38.755863 3898 base_network_controller_pods.go:487] [default/konflux-ui/proxy-7f4455bdf7-7qbcs] creating logical port konflux-ui_proxy-7f4455bdf7-7qbcs for pod on switch ip-10-0-0-48.ec2.internal I0327 23:12:38.755987 3898 kube.go:256] Updating pod konflux-ui/proxy-7f4455bdf7-7qbcs I0327 23:12:38.800415 3898 pod.go:62] [konflux-ui/proxy-7f4455bdf7-7qbcs] pod update took 44.475709ms I0327 23:12:38.800439 3898 base_network_controller_pods.go:937] [default/konflux-ui/proxy-7f4455bdf7-7qbcs] addLogicalPort annotation time took 44.505703ms I0327 23:12:38.801740 3898 pods.go:271] [konflux-ui/proxy-7f4455bdf7-7qbcs] addLogicalPort took 45.899045ms, libovsdb time 829.334µs I0327 23:12:38.822187 3898 namespace.go:142] [integration-service] updating namespace I0327 23:12:38.822240 3898 cni.go:403] [build-service/build-service-controller-manager-7df75dd4df-lv9t6 4053b9c3ae608a3a08559935de38b75361817b913c8b3065f8b218d43c3ff664 network default NAD default] ADD starting CNI request [build-service/build-service-controller-manager-7df75dd4df-lv9t6 4053b9c3ae608a3a08559935de38b75361817b913c8b3065f8b218d43c3ff664 network default NAD default] I0327 23:12:38.831254 3898 helper_linux.go:450] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7df75dd4df-lv9t6, hostIfaceName: 4053b9c3ae608a3, network: default, NAD default, SandboxID: "4053b9c3ae608a3a08559935de38b75361817b913c8b3065f8b218d43c3ff664", PCI device ID: , UID: "7e4dc434-f239-4791-b733-c01f89b84c97", MAC: 0a:58:0a:82:00:3b, IPs: [10.130.0.59/23] I0327 23:12:38.880975 3898 cni.go:424] [build-service/build-service-controller-manager-7df75dd4df-lv9t6 4053b9c3ae608a3a08559935de38b75361817b913c8b3065f8b218d43c3ff664 network default NAD default] ADD finished CNI request [build-service/build-service-controller-manager-7df75dd4df-lv9t6 4053b9c3ae608a3a08559935de38b75361817b913c8b3065f8b218d43c3ff664 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:ec:0d:3b:d3:1a\",\"name\":\"4053b9c3ae608a3\"},{\"mac\":\"0a:58:0a:82:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f51fee8b-da3e-4da7-8b39-5740a1b38d8d\"}],\"ips\":[{\"address\":\"10.130.0.59/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:12:38.909600 3898 cni.go:403] [konflux-ui/tsf-konflux-public-key-vpgth 7a7edf28a01186813a56dfd22019001d00e6fc4a9c4172f3f1ed756bb814d1b6 network default NAD default] DEL starting CNI request [konflux-ui/tsf-konflux-public-key-vpgth 7a7edf28a01186813a56dfd22019001d00e6fc4a9c4172f3f1ed756bb814d1b6 network default NAD default] I0327 23:12:38.985634 3898 cni.go:424] [konflux-ui/tsf-konflux-public-key-vpgth 7a7edf28a01186813a56dfd22019001d00e6fc4a9c4172f3f1ed756bb814d1b6 network default NAD default] DEL finished CNI request [konflux-ui/tsf-konflux-public-key-vpgth 7a7edf28a01186813a56dfd22019001d00e6fc4a9c4172f3f1ed756bb814d1b6 network default NAD default], result "{}", err I0327 23:12:39.031812 3898 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-vpgth of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:39.031845 3898 pods.go:176] Deleting pod: konflux-ui/tsf-konflux-public-key-vpgth I0327 23:12:39.033685 3898 pods.go:236] Attempting to release IPs for pod: konflux-ui/tsf-konflux-public-key-vpgth, ips: 10.130.0.57 I0327 23:12:39.033713 3898 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-vpgth of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:12:39.154629 3898 namespace.go:142] [konflux-info] updating namespace I0327 23:12:39.340509 3898 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:39.492828 3898 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 22904de078c1e59f12bf939f42c70f58f07bccabdc56d6a149f89347e5c59cfd network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 22904de078c1e59f12bf939f42c70f58f07bccabdc56d6a149f89347e5c59cfd network default NAD default] I0327 23:12:39.501442 3898 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-5f8cd64d55-jr9z8, hostIfaceName: 22904de078c1e59, network: default, NAD default, SandboxID: "22904de078c1e59f12bf939f42c70f58f07bccabdc56d6a149f89347e5c59cfd", PCI device ID: , UID: "7d3e6f58-5057-4bfa-a650-bfe672d5b2e9", MAC: 0a:58:0a:82:00:3a, IPs: [10.130.0.58/23] I0327 23:12:39.546443 3898 namespace.go:142] [release-service] updating namespace I0327 23:12:39.553558 3898 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 22904de078c1e59f12bf939f42c70f58f07bccabdc56d6a149f89347e5c59cfd network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 22904de078c1e59f12bf939f42c70f58f07bccabdc56d6a149f89347e5c59cfd network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:03:8f:bd:b7:c3\",\"name\":\"22904de078c1e59\"},{\"mac\":\"0a:58:0a:82:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd9411d4-0053-4823-894b-b8e6ccc99cd3\"}],\"ips\":[{\"address\":\"10.130.0.58/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 23:12:39.702243 3898 namespace.go:142] [build-service] updating namespace I0327 23:12:39.720450 3898 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:39.733282 3898 namespace.go:142] [image-controller] updating namespace I0327 23:12:39.746511 3898 namespace.go:142] [integration-service] updating namespace I0327 23:12:39.761753 3898 namespace.go:142] [konflux-info] updating namespace I0327 23:12:39.780200 3898 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:39.795990 3898 namespace.go:142] [release-service] updating namespace I0327 23:12:39.834716 3898 namespace.go:142] [build-service] updating namespace I0327 23:12:39.880724 3898 namespace.go:142] [enterprise-contract-service] updating namespace I0327 23:12:39.948777 3898 namespace.go:142] [image-controller] updating namespace I0327 23:12:40.047483 3898 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-vpgth of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:40.047516 3898 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-vpgth of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:12:40.050561 3898 namespace.go:142] [integration-service] updating namespace I0327 23:12:40.149369 3898 namespace.go:142] [konflux-info] updating namespace I0327 23:12:40.251927 3898 namespace.go:142] [namespace-lister] updating namespace I0327 23:12:40.345283 3898 namespace.go:142] [release-service] updating namespace I0327 23:12:41.536968 3898 namespace.go:142] [default-tenant] updating namespace I0327 23:12:41.738231 3898 namespace.go:142] [segment-bridge] updating namespace I0327 23:12:42.047972 3898 namespace.go:142] [default-tenant] updating namespace I0327 23:12:42.068923 3898 namespace.go:142] [segment-bridge] updating namespace I0327 23:12:42.170077 3898 namespace.go:142] [default-tenant] updating namespace I0327 23:12:42.268799 3898 namespace.go:142] [segment-bridge] updating namespace I0327 23:12:42.784568 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 W0327 23:12:42.930483 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 23:12:42.951821 3898 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 23:12:46.064708 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0327 23:12:46.619005 3898 cni.go:403] [konflux-ui/proxy-7f4455bdf7-7qbcs 0ceaa020f5945ac259aa1f5154191bad4eb57ff17d85843572d3f0a26171645c network default NAD default] ADD starting CNI request [konflux-ui/proxy-7f4455bdf7-7qbcs 0ceaa020f5945ac259aa1f5154191bad4eb57ff17d85843572d3f0a26171645c network default NAD default] I0327 23:12:46.627331 3898 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-7f4455bdf7-7qbcs, hostIfaceName: 0ceaa020f5945ac, network: default, NAD default, SandboxID: "0ceaa020f5945ac259aa1f5154191bad4eb57ff17d85843572d3f0a26171645c", PCI device ID: , UID: "4da58ef4-116b-4ce4-856b-571fb3ebe189", MAC: 0a:58:0a:82:00:3c, IPs: [10.130.0.60/23] I0327 23:12:46.674769 3898 cni.go:424] [konflux-ui/proxy-7f4455bdf7-7qbcs 0ceaa020f5945ac259aa1f5154191bad4eb57ff17d85843572d3f0a26171645c network default NAD default] ADD finished CNI request [konflux-ui/proxy-7f4455bdf7-7qbcs 0ceaa020f5945ac259aa1f5154191bad4eb57ff17d85843572d3f0a26171645c network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:33:8c:85:6e:6a\",\"name\":\"0ceaa020f5945ac\"},{\"mac\":\"0a:58:0a:82:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1c00b419-77fd-459c-8547-452d49d68924\"}],\"ips\":[{\"address\":\"10.130.0.60/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 23:12:52.001506 3898 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 23:12:52.001529 3898 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 23:12:53.942960 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 23:12:53.970570 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 23:12:56.139551 3898 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0327 23:12:56.157993 3898 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 23:12:56.161311 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 23:12:56.186031 3898 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 23:12:56.186049 3898 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 23:12:56.204911 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 23:12:56.208337 3898 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 23:12:56.234609 3898 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0327 23:12:56.234760 3898 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0327 23:13:07.489216 3898 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-c8pzp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:07.489842 3898 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-c8pzp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:07.757819 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0327 23:13:08.242148 3898 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-c8pzp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:08.242181 3898 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-c8pzp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:08.247087 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-c8pzp I0327 23:13:08.247114 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-c8pzp W0327 23:13:29.061556 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0327 23:13:29.061577 3898 util.go:959] Failed to build node endpoints for node ip-10-0-0-48.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0327 23:13:59.358826 3898 obj_retry.go:464] Detected object konflux-ui/proxy-58bb569fbd-fksns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:59.359405 3898 obj_retry.go:464] Detected object konflux-ui/proxy-58bb569fbd-fksns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:13:59.536258 3898 obj_retry.go:458] Detected object konflux-ui/proxy-58bb569fbd-fksns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:59.536284 3898 obj_retry.go:458] Detected object konflux-ui/proxy-58bb569fbd-fksns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:13:59.540335 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-58bb569fbd-fksns I0327 23:13:59.540357 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-58bb569fbd-fksns I0327 23:14:03.785202 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 I0327 23:14:10.745159 3898 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:10.745710 3898 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:10.789093 3898 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:10.789118 3898 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:10.793211 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0327 23:14:10.793238 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0327 23:14:17.751526 3898 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0327 23:14:24.664622 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=52 I0327 23:14:40.751422 3898 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 5663de9fc0cc7d78ef02a7486973a1cb55d18616b77a238b630bc607e87658cd network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 5663de9fc0cc7d78ef02a7486973a1cb55d18616b77a238b630bc607e87658cd network default NAD default] I0327 23:14:40.838027 3898 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 5663de9fc0cc7d78ef02a7486973a1cb55d18616b77a238b630bc607e87658cd network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 5663de9fc0cc7d78ef02a7486973a1cb55d18616b77a238b630bc607e87658cd network default NAD default], result "{}", err I0327 23:14:40.904368 3898 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:40.904407 3898 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 I0327 23:14:40.906015 3898 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-pss55, ips: 10.130.0.34 I0327 23:14:40.906044 3898 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:41.616884 3898 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:41.616911 3898 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:41.622253 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 I0327 23:14:41.622275 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-pss55 I0327 23:14:43.675239 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=156 I0327 23:14:44.666596 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=66 I0327 23:14:55.984128 3898 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 22904de078c1e59f12bf939f42c70f58f07bccabdc56d6a149f89347e5c59cfd network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 22904de078c1e59f12bf939f42c70f58f07bccabdc56d6a149f89347e5c59cfd network default NAD default] I0327 23:14:56.063964 3898 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 22904de078c1e59f12bf939f42c70f58f07bccabdc56d6a149f89347e5c59cfd network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 22904de078c1e59f12bf939f42c70f58f07bccabdc56d6a149f89347e5c59cfd network default NAD default], result "{}", err I0327 23:14:56.115282 3898 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:56.115322 3898 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 I0327 23:14:56.116883 3898 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8, ips: 10.130.0.58 I0327 23:14:56.116913 3898 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 23:14:56.703133 3898 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:56.703160 3898 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 23:14:56.705994 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 I0327 23:14:56.706025 3898 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-jr9z8 I0327 23:15:23.675906 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=498 I0327 23:15:37.663765 3898 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=76 I0327 23:16:30.746942 3898 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9