+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-80.ec2.internal ++ [[ -n ip-10-0-0-80.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-80.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0327 15:03:44.507502559 - disable conntrack on geneve port' I0327 15:03:44.507502559 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0327 15:03:44.559434024 - starting ovnkube-node + echo 'I0327 15:03:44.559434024 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-80.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-80.ec2.internal --init-node ip-10-0-0-80.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-80.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0327 15:03:44.611648 3944 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0327 15:03:44.611704 3944 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-c3e8d4b4b8.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0327 15:03:44.613227 3944 kube.go:419] Waiting for certificate I0327 15:03:44.613286 3944 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0327 15:03:44.613321 3944 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0327 15:03:44.614441 3944 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 15:03:44.637646 3944 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:03:44.637681 3944 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:03:44.640011 3944 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:03:44.642347 3944 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-7zgx8" I0327 15:03:44.650942 3944 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-7zgx8" I0327 15:03:44.651065 3944 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0327 15:03:45.613353 3944 kube.go:426] Certificate found I0327 15:03:45.614010 3944 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0327 15:03:45.614896 3944 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0327 15:03:45.663525 3944 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-28 14:58:44 +0000 UTC" deadline="2026-03-28 11:58:02.632211928 +0000 UTC" I0327 15:03:45.663830 3944 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h54m16.968493043s" I0327 15:03:45.684818 3944 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 15:03:45.696492 3944 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0327 15:03:45.700565 3944 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0327 15:03:45.704599 3944 factory.go:531] Starting watch factory I0327 15:03:45.704881 3944 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.704909 3944 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.705044 3944 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.705129 3944 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.705257 3944 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.705402 3944 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.705543 3944 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.705571 3944 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.705577 3944 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.705609 3944 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.707764 3944 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.707787 3944 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.711193 3944 controller_manager.go:368] Starting the ovnkube controller I0327 15:03:45.711252 3944 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-80.ec2.internal I0327 15:03:45.711333 3944 controller_manager.go:393] NBDB zone sync took: 64.22µs I0327 15:03:45.711348 3944 factory.go:531] Starting watch factory I0327 15:03:45.722953 3944 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.723302 3944 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.723377 3944 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.723927 3944 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.724173 3944 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.732312 3944 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0327 15:03:45.732985 3944 ovn_northd.go:121] Registering OVN northd metrics I0327 15:03:45.735751 3944 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0327 15:03:45.735807 3944 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0327 15:03:45.738113 3944 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0327 15:03:45.751781 3944 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0327 15:03:45.809883 3944 factory.go:1890] *v1.Service informer cache synced successfully I0327 15:03:45.809904 3944 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 15:03:45.809908 3944 factory.go:1890] *v1.Pod informer cache synced successfully I0327 15:03:45.809911 3944 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 15:03:45.809916 3944 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 15:03:45.809919 3944 factory.go:1890] *v1.Node informer cache synced successfully I0327 15:03:45.810032 3944 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:45.810054 3944 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:45.810098 3944 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:45.810115 3944 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:45.812158 3944 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:45.812161 3944 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0327 15:03:45.815713 3944 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 15:03:45.815727 3944 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 15:03:45.815791 3944 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.815803 3944 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.817185 3944 factory.go:1890] *v1.Service informer cache synced successfully I0327 15:03:45.817196 3944 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0327 15:03:45.817200 3944 factory.go:1890] *v1.Pod informer cache synced successfully I0327 15:03:45.817203 3944 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0327 15:03:45.817206 3944 factory.go:1890] *v1.Namespace informer cache synced successfully I0327 15:03:45.817209 3944 factory.go:1890] *v1.Node informer cache synced successfully I0327 15:03:45.817842 3944 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.821257 3944 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 15:03:45.821321 3944 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.821333 3944 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.822612 3944 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0327 15:03:45.822623 3944 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0327 15:03:45.822914 3944 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.827278 3944 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 15:03:45.827369 3944 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.827390 3944 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.827643 3944 factory.go:1890] *v1.EgressIP informer cache synced successfully I0327 15:03:45.829069 3944 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.832365 3944 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 15:03:45.832446 3944 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.832462 3944 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.832706 3944 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0327 15:03:45.834017 3944 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.838357 3944 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0327 15:03:45.838371 3944 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 15:03:45.838447 3944 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.838466 3944 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.840023 3944 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.844421 3944 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 15:03:45.844431 3944 factory.go:1890] *v1.EgressService informer cache synced successfully I0327 15:03:45.844654 3944 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:03:45.844688 3944 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:03:45.846431 3944 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0327 15:03:45.849856 3944 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 15:03:45.849871 3944 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0327 15:03:45.849951 3944 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.849969 3944 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.849997 3944 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.850012 3944 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.851645 3944 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.851976 3944 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0327 15:03:45.855551 3944 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0327 15:03:45.855568 3944 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 15:03:45.855578 3944 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 15:03:45.855584 3944 factory.go:636] Watch Factory start up complete, took: 150.995087ms I0327 15:03:45.855650 3944 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0327 15:03:45.861011 3944 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0327 15:03:45.861029 3944 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0327 15:03:45.861038 3944 factory.go:636] Watch Factory start up complete, took: 149.687885ms I0327 15:03:45.861048 3944 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-0-80.ec2.internal" zone I0327 15:03:45.908777 3944 default_node_network_controller.go:709] Initializing the default node network controller I0327 15:03:45.977346 3944 udn_isolation.go:101] Starting UDN host isolation manager I0327 15:03:45.984877 3944 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0327 15:03:46.073522 3944 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0327 15:03:46.073646 3944 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0327 15:03:46.073682 3944 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0327 15:03:46.111200 3944 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0327 15:03:46.111268 3944 default_node_network_controller.go:808] Node ip-10-0-0-80.ec2.internal ready for ovn initialization with subnet 10.130.0.0/23 I0327 15:03:46.117544 3944 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.80"] k8s.ovn.org/zone-name:ip-10-0-0-80.ec2.internal] on node ip-10-0-0-80.ec2.internal I0327 15:03:46.136448 3944 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0327 15:03:46.142288 3944 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0327 15:03:46.142426 3944 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0327 15:03:46.142575 3944 gateway_init.go:263] Preparing Gateway I0327 15:03:46.142587 3944 gateway_shared_intf.go:1658] Creating new gateway I0327 15:03:46.217633 3944 iptables.go:108] Creating table: filter chain: FORWARD I0327 15:03:46.223964 3944 iptables.go:108] Creating table: filter chain: OUTPUT I0327 15:03:46.230236 3944 gateway_shared_intf.go:1787] Gateway Creation Complete I0327 15:03:46.230256 3944 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-80.ec2.internal","mac-address":"12:78:66:10:09:29","ip-addresses":["10.0.0.80/24"],"ip-address":"10.0.0.80/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:52e1306a-bd92-4ed7-8b03-3cc58b56001f k8s.ovn.org/node-encap-ips:["10.0.0.80"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.80/24"} k8s.ovn.org/zone-name:ip-10-0-0-80.ec2.internal] on node ip-10-0-0-80.ec2.internal I0327 15:03:46.246565 3944 default_node_network_controller.go:518] Node connection status = connected I0327 15:03:46.361596 3944 controller_manager.go:427] Waiting for node in zone sync took: 500.532017ms I0327 15:03:46.365629 3944 controller_manager.go:608] Switching to transit router for layer2 networks I0327 15:03:46.365645 3944 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-80.ec2.internal I0327 15:03:46.380088 3944 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0327 15:03:46.386600 3944 controller_manager.go:288] SCTP support detected in OVN I0327 15:03:46.387245 3944 services_controller.go:69] Creating services controller for network=default I0327 15:03:46.387337 3944 udn_enabled_svc.go:80] Waiting for service informer to sync I0327 15:03:46.387418 3944 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0327 15:03:46.387539 3944 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0327 15:03:46.387561 3944 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0327 15:03:46.387572 3944 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 15:03:46.387765 3944 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0327 15:03:46.387815 3944 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0327 15:03:46.387852 3944 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 26.986µs I0327 15:03:46.387863 3944 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 48.489µs I0327 15:03:46.387864 3944 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 58.647µs I0327 15:03:46.387891 3944 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0327 15:03:46.387897 3944 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0327 15:03:46.387899 3944 udn_enabled_svc.go:95] Performing full resync I0327 15:03:46.387905 3944 default_network_controller.go:321] Starting the default network controller I0327 15:03:46.388085 3944 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 14.129µs I0327 15:03:46.388196 3944 udn_enabled_svc.go:99] Waiting for handler to sync I0327 15:03:46.388206 3944 udn_enabled_svc.go:104] Starting worker I0327 15:03:46.397216 3944 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0327 15:03:46.397275 3944 repair.go:33] Syncing exgw routes took 44.737µs I0327 15:03:46.397296 3944 default_network_controller.go:407] Starting all the Watchers... I0327 15:03:46.397449 3944 namespace.go:98] [openshift-ingress] adding namespace I0327 15:03:46.397460 3944 namespace.go:98] [openshift-dns-operator] adding namespace I0327 15:03:46.397463 3944 namespace.go:98] [openshift-backplane-cee] adding namespace I0327 15:03:46.397467 3944 namespace.go:98] [openshift-service-ca-operator] adding namespace I0327 15:03:46.397451 3944 namespace.go:98] [default] adding namespace I0327 15:03:46.397480 3944 namespace.go:98] [openshift-etcd] adding namespace I0327 15:03:46.397489 3944 namespace.go:98] [openshift-config] adding namespace I0327 15:03:46.397491 3944 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0327 15:03:46.397496 3944 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0327 15:03:46.397499 3944 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0327 15:03:46.397494 3944 namespace.go:98] [openshift-ingress-canary] adding namespace I0327 15:03:46.397510 3944 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0327 15:03:46.397515 3944 namespace.go:98] [openshift-backplane-tam] adding namespace I0327 15:03:46.397525 3944 namespace.go:98] [kube-system] adding namespace I0327 15:03:46.397449 3944 namespace.go:98] [openshift-config-operator] adding namespace I0327 15:03:46.398113 3944 namespace.go:102] [openshift-ingress] adding namespace took 650.348µs I0327 15:03:46.398134 3944 namespace.go:98] [kube-public] adding namespace I0327 15:03:46.399124 3944 namespace.go:102] [kube-public] adding namespace took 982.505µs I0327 15:03:46.399141 3944 namespace.go:98] [openshift-dns] adding namespace I0327 15:03:46.399824 3944 namespace.go:102] [openshift-dns] adding namespace took 675.706µs I0327 15:03:46.399841 3944 namespace.go:98] [openshift-marketplace] adding namespace I0327 15:03:46.400320 3944 namespace.go:102] [openshift-dns-operator] adding namespace took 2.850752ms I0327 15:03:46.400335 3944 namespace.go:98] [openshift-backplane-csm] adding namespace I0327 15:03:46.400793 3944 namespace.go:102] [openshift-backplane-cee] adding namespace took 3.31765ms I0327 15:03:46.400808 3944 namespace.go:98] [openshift-route-controller-manager] adding namespace I0327 15:03:46.401369 3944 namespace.go:102] [openshift-service-ca-operator] adding namespace took 3.887506ms I0327 15:03:46.401384 3944 namespace.go:98] [openshift-backplane-mobb] adding namespace I0327 15:03:46.401973 3944 namespace.go:102] [default] adding namespace took 4.486681ms I0327 15:03:46.401992 3944 namespace.go:98] [openshift-config-managed] adding namespace I0327 15:03:46.402473 3944 namespace.go:102] [openshift-etcd] adding namespace took 4.987208ms I0327 15:03:46.402483 3944 namespace.go:98] [openshift-controller-manager] adding namespace I0327 15:03:46.402963 3944 namespace.go:102] [openshift-config] adding namespace took 5.463792ms I0327 15:03:46.402975 3944 namespace.go:98] [openshift-customer-monitoring] adding namespace I0327 15:03:46.403379 3944 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 5.878829ms I0327 15:03:46.403388 3944 namespace.go:98] [openshift-console-operator] adding namespace I0327 15:03:46.403956 3944 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.451953ms I0327 15:03:46.403972 3944 namespace.go:98] [openshift-network-console] adding namespace I0327 15:03:46.404466 3944 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.958006ms I0327 15:03:46.404486 3944 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0327 15:03:46.405012 3944 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.502934ms I0327 15:03:46.405029 3944 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0327 15:03:46.405601 3944 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 8.083504ms I0327 15:03:46.405612 3944 namespace.go:98] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] adding namespace I0327 15:03:46.406213 3944 namespace.go:102] [openshift-backplane-tam] adding namespace took 8.690584ms I0327 15:03:46.406226 3944 namespace.go:98] [openshift-kube-apiserver] adding namespace I0327 15:03:46.406868 3944 namespace.go:102] [kube-system] adding namespace took 9.33745ms I0327 15:03:46.406879 3944 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0327 15:03:46.407535 3944 namespace.go:102] [openshift-config-operator] adding namespace took 9.958667ms I0327 15:03:46.407550 3944 namespace.go:98] [openshift-machine-api] adding namespace I0327 15:03:46.408070 3944 namespace.go:102] [openshift-marketplace] adding namespace took 8.22312ms I0327 15:03:46.408084 3944 namespace.go:98] [openshift-network-node-identity] adding namespace I0327 15:03:46.408502 3944 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.162335ms I0327 15:03:46.408514 3944 namespace.go:98] [openshift-image-registry] adding namespace I0327 15:03:46.409035 3944 namespace.go:102] [openshift-route-controller-manager] adding namespace took 8.222658ms I0327 15:03:46.409047 3944 namespace.go:98] [openshift-logging] adding namespace I0327 15:03:46.409473 3944 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.084315ms I0327 15:03:46.409487 3944 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0327 15:03:46.409860 3944 namespace.go:102] [openshift-config-managed] adding namespace took 7.863551ms I0327 15:03:46.409875 3944 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0327 15:03:46.410314 3944 namespace.go:102] [openshift-controller-manager] adding namespace took 7.82493ms I0327 15:03:46.410327 3944 namespace.go:98] [openshift-machine-config-operator] adding namespace I0327 15:03:46.410886 3944 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.904603ms I0327 15:03:46.410901 3944 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0327 15:03:46.411341 3944 namespace.go:102] [openshift-console-operator] adding namespace took 7.947271ms I0327 15:03:46.411355 3944 namespace.go:98] [openshift-apiserver] adding namespace I0327 15:03:46.411853 3944 namespace.go:102] [openshift-network-console] adding namespace took 7.876683ms I0327 15:03:46.411866 3944 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0327 15:03:46.412285 3944 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.7908ms I0327 15:03:46.412303 3944 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0327 15:03:46.412688 3944 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.654475ms I0327 15:03:46.412700 3944 namespace.go:98] [openshift-insights] adding namespace I0327 15:03:46.413185 3944 namespace.go:102] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] adding namespace took 7.568294ms I0327 15:03:46.413198 3944 namespace.go:98] [openshift-authentication-operator] adding namespace I0327 15:03:46.413629 3944 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.396921ms I0327 15:03:46.413646 3944 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0327 15:03:46.414176 3944 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.29203ms I0327 15:03:46.414187 3944 namespace.go:98] [openshift-network-diagnostics] adding namespace I0327 15:03:46.414640 3944 namespace.go:102] [openshift-machine-api] adding namespace took 7.084757ms I0327 15:03:46.414651 3944 namespace.go:98] [openshift-backplane-srep] adding namespace I0327 15:03:46.415097 3944 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.009148ms I0327 15:03:46.415547 3944 namespace.go:102] [openshift-image-registry] adding namespace took 7.027322ms I0327 15:03:46.415561 3944 namespace.go:98] [openshift] adding namespace I0327 15:03:46.416039 3944 namespace.go:102] [openshift-logging] adding namespace took 6.987066ms I0327 15:03:46.416055 3944 namespace.go:98] [openshift-multus] adding namespace I0327 15:03:46.416490 3944 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.997818ms I0327 15:03:46.416506 3944 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0327 15:03:46.416992 3944 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.112649ms I0327 15:03:46.417004 3944 namespace.go:98] [openshift-cluster-version] adding namespace I0327 15:03:46.417555 3944 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.219907ms I0327 15:03:46.417577 3944 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0327 15:03:46.418011 3944 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.105821ms I0327 15:03:46.418022 3944 namespace.go:98] [openshift-node] adding namespace I0327 15:03:46.418446 3944 namespace.go:102] [openshift-apiserver] adding namespace took 7.084634ms I0327 15:03:46.418463 3944 namespace.go:98] [openshift-network-operator] adding namespace I0327 15:03:46.418894 3944 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.020664ms I0327 15:03:46.418911 3944 namespace.go:98] [openshift-console] adding namespace I0327 15:03:46.419855 3944 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.54456ms I0327 15:03:46.419871 3944 namespace.go:98] [kube-node-lease] adding namespace I0327 15:03:46.420402 3944 namespace.go:102] [openshift-insights] adding namespace took 7.697322ms I0327 15:03:46.420414 3944 namespace.go:98] [openshift-operators] adding namespace I0327 15:03:46.420929 3944 namespace.go:102] [openshift-authentication-operator] adding namespace took 7.72453ms I0327 15:03:46.420945 3944 namespace.go:98] [dedicated-admin] adding namespace W0327 15:03:46.421422 3944 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-9q9gv: pod openshift-network-diagnostics/network-check-target-9q9gv: no pod IPs found I0327 15:03:46.421464 3944 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.810212ms I0327 15:03:46.421482 3944 namespace.go:98] [openshift-backplane-cse] adding namespace I0327 15:03:46.421953 3944 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.759677ms I0327 15:03:46.421971 3944 namespace.go:98] [openshift-package-operator] adding namespace I0327 15:03:46.422534 3944 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.878489ms I0327 15:03:46.422545 3944 namespace.go:98] [openshift-host-network] adding namespace W0327 15:03:46.423035 3944 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-ktn4j: pod openshift-multus/network-metrics-daemon-ktn4j: no pod IPs found I0327 15:03:46.423056 3944 namespace.go:102] [openshift] adding namespace took 7.490509ms I0327 15:03:46.423066 3944 namespace.go:98] [openshift-console-user-settings] adding namespace I0327 15:03:46.423625 3944 namespace.go:102] [openshift-multus] adding namespace took 7.566112ms I0327 15:03:46.423635 3944 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0327 15:03:46.424097 3944 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.587401ms I0327 15:03:46.424107 3944 namespace.go:98] [openshift-infra] adding namespace I0327 15:03:46.424524 3944 namespace.go:102] [openshift-cluster-version] adding namespace took 7.515735ms I0327 15:03:46.424533 3944 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0327 15:03:46.425053 3944 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 7.469049ms I0327 15:03:46.425072 3944 namespace.go:98] [openshift-apiserver-operator] adding namespace I0327 15:03:46.425522 3944 namespace.go:102] [openshift-node] adding namespace took 7.494392ms I0327 15:03:46.425534 3944 namespace.go:98] [openshift-backplane] adding namespace I0327 15:03:46.426030 3944 namespace.go:102] [openshift-network-operator] adding namespace took 7.560547ms I0327 15:03:46.426044 3944 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0327 15:03:46.426515 3944 namespace.go:102] [openshift-console] adding namespace took 7.598859ms I0327 15:03:46.426530 3944 namespace.go:98] [openshift-monitoring] adding namespace I0327 15:03:46.426959 3944 namespace.go:102] [kube-node-lease] adding namespace took 7.082468ms I0327 15:03:46.426973 3944 namespace.go:98] [openshift-ingress-operator] adding namespace I0327 15:03:46.427371 3944 namespace.go:102] [openshift-operators] adding namespace took 6.952299ms I0327 15:03:46.427382 3944 namespace.go:98] [openshift-operators-redhat] adding namespace I0327 15:03:46.427806 3944 namespace.go:102] [dedicated-admin] adding namespace took 6.856693ms I0327 15:03:46.427818 3944 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0327 15:03:46.428299 3944 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.813145ms I0327 15:03:46.428308 3944 namespace.go:98] [openshift-must-gather-operator] adding namespace I0327 15:03:46.428827 3944 namespace.go:102] [openshift-package-operator] adding namespace took 6.847599ms I0327 15:03:46.428845 3944 namespace.go:98] [openshift-kube-scheduler] adding namespace I0327 15:03:46.429361 3944 namespace.go:102] [openshift-host-network] adding namespace took 6.810681ms I0327 15:03:46.429375 3944 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0327 15:03:46.429749 3944 namespace.go:102] [openshift-console-user-settings] adding namespace took 6.677362ms I0327 15:03:46.430225 3944 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.5859ms I0327 15:03:46.430238 3944 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0327 15:03:46.430570 3944 namespace.go:102] [openshift-infra] adding namespace took 6.459092ms I0327 15:03:46.430581 3944 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0327 15:03:46.431039 3944 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.501739ms I0327 15:03:46.431051 3944 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0327 15:03:46.431376 3944 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.298246ms I0327 15:03:46.431809 3944 namespace.go:102] [openshift-backplane] adding namespace took 6.269721ms I0327 15:03:46.432189 3944 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.141242ms I0327 15:03:46.432200 3944 namespace.go:98] [openshift-authentication] adding namespace I0327 15:03:46.432626 3944 namespace.go:102] [openshift-monitoring] adding namespace took 6.089886ms I0327 15:03:46.433087 3944 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.109354ms I0327 15:03:46.433525 3944 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.137988ms I0327 15:03:46.433879 3944 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.056238ms I0327 15:03:46.434340 3944 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.02627ms I0327 15:03:46.434797 3944 namespace.go:102] [openshift-kube-scheduler] adding namespace took 5.947223ms I0327 15:03:46.435214 3944 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.834788ms I0327 15:03:46.435603 3944 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 5.361027ms I0327 15:03:46.435995 3944 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 5.409694ms I0327 15:03:46.436391 3944 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 5.335501ms I0327 15:03:46.436734 3944 namespace.go:102] [openshift-authentication] adding namespace took 4.529649ms I0327 15:03:46.437099 3944 zone_ic_handler.go:186] Time taken to create transit switch: 263.445µs I0327 15:03:46.437192 3944 master.go:532] Adding or Updating local node "ip-10-0-0-80.ec2.internal" for network "default" I0327 15:03:46.437629 3944 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-176.ec2.internal for the network default I0327 15:03:46.437841 3944 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-39.ec2.internal for the network default I0327 15:03:46.438630 3944 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-176.ec2.internal" on network "default" took: 1.013078ms I0327 15:03:46.438870 3944 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-39.ec2.internal" on network "default" took: 1.039185ms W0327 15:03:46.445129 3944 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-9q9gv: pod openshift-network-diagnostics/network-check-target-9q9gv: no pod IPs found W0327 15:03:46.445147 3944 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-ktn4j: pod openshift-multus/network-metrics-daemon-ktn4j: no pod IPs found I0327 15:03:46.446023 3944 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-0-80.ec2.internal for the network default I0327 15:03:46.446736 3944 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0327 15:03:46.446765 3944 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0327 15:03:46.446779 3944 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0327 15:03:46.446804 3944 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-80.ec2.internal in network "default" I0327 15:03:46.446848 3944 node_tracker.go:169] Node ip-10-0-0-80.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:46.446858 3944 services_controller.go:574] Full service sync requested for network=default I0327 15:03:46.446863 3944 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-176.ec2.internal in network "default" I0327 15:03:46.446885 3944 node_tracker.go:169] Node ip-10-0-1-176.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:46.446909 3944 services_controller.go:574] Full service sync requested for network=default I0327 15:03:46.446919 3944 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-39.ec2.internal in network "default" I0327 15:03:46.446939 3944 node_tracker.go:169] Node ip-10-0-2-39.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:46.446946 3944 services_controller.go:574] Full service sync requested for network=default I0327 15:03:46.547510 3944 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0327 15:03:46.547538 3944 services_controller.go:208] Setting up event handlers for services for network=default I0327 15:03:46.547611 3944 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0327 15:03:46.547661 3944 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0327 15:03:46.547692 3944 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0327 15:03:46.647916 3944 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0327 15:03:46.647957 3944 repair.go:60] Starting repairing loop for services I0327 15:03:46.648061 3944 repair.go:131] Deleted 0 stale service LBs I0327 15:03:46.648088 3944 repair.go:137] Deleted 0 stale Chassis Template Vars I0327 15:03:46.648110 3944 repair.go:62] Finished repairing loop for services: 154.209µs I0327 15:03:46.648145 3944 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0327 15:03:46.648158 3944 services_controller.go:254] Starting workers for network=default I0327 15:03:46.648643 3944 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-9q9gv] creating logical port openshift-network-diagnostics_network-check-target-9q9gv for pod on switch ip-10-0-0-80.ec2.internal I0327 15:03:46.648714 3944 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-ktn4j] creating logical port openshift-multus_network-metrics-daemon-ktn4j for pod on switch ip-10-0-0-80.ec2.internal I0327 15:03:46.648778 3944 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-9q9gv I0327 15:03:46.648864 3944 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-ktn4j W0327 15:03:46.651936 3944 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:03:46.663828 3944 pod.go:62] [openshift-network-diagnostics/network-check-target-9q9gv] pod update took 15.091123ms I0327 15:03:46.663852 3944 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-9q9gv] addLogicalPort annotation time took 15.11816ms I0327 15:03:46.664592 3944 pod.go:62] [openshift-multus/network-metrics-daemon-ktn4j] pod update took 15.797207ms I0327 15:03:46.664608 3944 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-ktn4j] addLogicalPort annotation time took 15.81736ms I0327 15:03:46.664798 3944 pods.go:271] [openshift-network-diagnostics/network-check-target-9q9gv] addLogicalPort took 16.160839ms, libovsdb time 703.674µs I0327 15:03:46.665709 3944 pods.go:271] [openshift-multus/network-metrics-daemon-ktn4j] addLogicalPort took 17.005146ms, libovsdb time 768.671µs I0327 15:03:46.666263 3944 admin_network_policy_controller.go:227] Starting controller default-network-controller I0327 15:03:46.666287 3944 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0327 15:03:46.666303 3944 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0327 15:03:46.666309 3944 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0327 15:03:46.666516 3944 repair.go:29] Repairing admin network policies took 199.698µs I0327 15:03:46.666709 3944 repair.go:92] Repairing baseline admin network policies took 179.797µs I0327 15:03:46.667429 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0327 15:03:46.667454 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0327 15:03:46.669423 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0327 15:03:46.669449 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0327 15:03:46.669459 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0327 15:03:46.671101 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0327 15:03:46.671126 3944 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0327 15:03:46.671144 3944 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0327 15:03:46.674037 3944 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0327 15:03:46.674060 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0327 15:03:46.674072 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0327 15:03:46.674663 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0327 15:03:46.674702 3944 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0327 15:03:46.674717 3944 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0327 15:03:46.677337 3944 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0327 15:03:46.677355 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0327 15:03:46.677364 3944 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0327 15:03:46.679747 3944 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0327 15:03:46.679767 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0327 15:03:46.679775 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0327 15:03:46.680407 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0327 15:03:46.680429 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0327 15:03:46.680439 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0327 15:03:46.682258 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0327 15:03:46.682281 3944 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0327 15:03:46.682293 3944 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0327 15:03:46.683357 3944 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0327 15:03:46.683382 3944 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0327 15:03:46.683392 3944 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0327 15:03:46.685221 3944 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0327 15:03:46.685241 3944 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0327 15:03:46.685251 3944 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0327 15:03:46.686571 3944 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0327 15:03:46.686595 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0327 15:03:46.686606 3944 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0327 15:03:46.687236 3944 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0327 15:03:46.687254 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0327 15:03:46.687261 3944 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0327 15:03:46.687878 3944 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0327 15:03:46.687896 3944 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0327 15:03:46.687904 3944 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0327 15:03:46.688601 3944 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0327 15:03:46.688624 3944 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0327 15:03:46.688650 3944 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0327 15:03:46.689966 3944 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0327 15:03:46.689985 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0327 15:03:46.689993 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0327 15:03:46.690602 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0327 15:03:46.690625 3944 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0327 15:03:46.690635 3944 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0327 15:03:46.691957 3944 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0327 15:03:46.691979 3944 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0327 15:03:46.691990 3944 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0327 15:03:46.693614 3944 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0327 15:03:46.693638 3944 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0327 15:03:46.693649 3944 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0327 15:03:46.695781 3944 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0327 15:03:46.695799 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0327 15:03:46.696696 3944 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0327 15:03:46.696709 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 15:03:46.697518 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0327 15:03:46.697537 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0327 15:03:46.697544 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0327 15:03:46.697904 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0327 15:03:46.697929 3944 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0327 15:03:46.697938 3944 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0327 15:03:46.698244 3944 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0327 15:03:46.698263 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0327 15:03:46.698271 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0327 15:03:46.698837 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0327 15:03:46.698872 3944 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0327 15:03:46.698898 3944 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0327 15:03:46.700549 3944 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0327 15:03:46.700572 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0327 15:03:46.700607 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0327 15:03:46.701585 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0327 15:03:46.701608 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0327 15:03:46.701620 3944 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0327 15:03:46.702238 3944 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0327 15:03:46.702260 3944 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0327 15:03:46.702277 3944 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0327 15:03:46.702837 3944 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0327 15:03:46.702862 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0327 15:03:46.702876 3944 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0327 15:03:46.703562 3944 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0327 15:03:46.703580 3944 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0327 15:03:46.703589 3944 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0327 15:03:46.703910 3944 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0327 15:03:46.710443 3944 egressfirewall.go:319] Starting EgressFirewall controller I0327 15:03:46.710474 3944 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0327 15:03:46.710490 3944 controller.go:133] Adding controller egress-firewall-controller event handlers I0327 15:03:46.710503 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0327 15:03:46.710519 3944 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0327 15:03:46.710545 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0327 15:03:46.710557 3944 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0327 15:03:46.710840 3944 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0327 15:03:46.710853 3944 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0327 15:03:46.710862 3944 egressqos.go:193] Setting up event handlers for EgressQoS I0327 15:03:46.710943 3944 egressfirewall.go:628] Syncing node "ip-10-0-2-39.ec2.internal" for egress firewall I0327 15:03:46.710953 3944 egressfirewall.go:628] Syncing node "ip-10-0-0-80.ec2.internal" for egress firewall I0327 15:03:46.710957 3944 egressfirewall.go:628] Syncing node "ip-10-0-1-176.ec2.internal" for egress firewall I0327 15:03:46.710960 3944 egressqos.go:245] Starting EgressQoS Controller I0327 15:03:46.710968 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0327 15:03:46.710974 3944 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0327 15:03:46.710979 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0327 15:03:46.710984 3944 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0327 15:03:46.710989 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0327 15:03:46.710993 3944 shared_informer.go:356] "Caches are synced" controller="egressqos" I0327 15:03:46.710997 3944 egressqos.go:259] Repairing EgressQoSes I0327 15:03:46.711000 3944 egressqos.go:399] Starting repairing loop for egressqos I0327 15:03:46.711075 3944 egressqos.go:401] Finished repairing loop for egressqos: 74.062µs I0327 15:03:46.711092 3944 egressservice_zone.go:132] Setting up event handlers for Egress Services I0327 15:03:46.711121 3944 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-80.ec2.internal I0327 15:03:46.711183 3944 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-80.ec2.internal : 62.982µs I0327 15:03:46.711808 3944 egressservice_zone.go:210] Starting Egress Services Controller I0327 15:03:46.711828 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 15:03:46.711838 3944 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 15:03:46.711844 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 15:03:46.711849 3944 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 15:03:46.711855 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 15:03:46.711860 3944 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 15:03:46.711864 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0327 15:03:46.711869 3944 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0327 15:03:46.711872 3944 egressservice_zone.go:228] Repairing Egress Services I0327 15:03:46.712188 3944 master_controller.go:88] Starting Admin Policy Based Route Controller I0327 15:03:46.712198 3944 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 15:03:46.712267 3944 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:03:46.712283 3944 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 18.064µs I0327 15:03:46.712294 3944 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-176.ec2.internal I0327 15:03:46.712305 3944 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-176.ec2.internal: 12.104µs I0327 15:03:46.712310 3944 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-39.ec2.internal I0327 15:03:46.712315 3944 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-39.ec2.internal: 5.461µs I0327 15:03:46.712329 3944 default_network_controller.go:568] Completing all the Watchers took 315.021729ms I0327 15:03:46.712345 3944 default_network_controller.go:572] Starting unidling controllers I0327 15:03:46.712360 3944 unidle.go:46] Registering OVN SB ControllerEvent handler I0327 15:03:46.712368 3944 unidle.go:63] Populating Initial ContollerEvent events I0327 15:03:46.712382 3944 unidle.go:79] Setting up event handlers for services I0327 15:03:46.712446 3944 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0327 15:03:46.746903 3944 default_node_network_controller.go:518] Node connection status = connected I0327 15:03:47.246992 3944 default_node_network_controller.go:518] Node connection status = connected W0327 15:03:47.594980 3944 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:03:47.747503 3944 default_node_network_controller.go:518] Node connection status = connected I0327 15:03:48.247188 3944 default_node_network_controller.go:518] Node connection status = connected I0327 15:03:48.507819 3944 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0327 15:03:48.747376 3944 default_node_network_controller.go:518] Node connection status = connected I0327 15:03:48.765932 3944 bridgeconfig.go:537] Gateway is ready I0327 15:03:48.765964 3944 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0327 15:03:48.778124 3944 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.80. Updating OVN encap IP. I0327 15:03:48.783220 3944 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.80 - it is already configured I0327 15:03:48.783237 3944 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.80/24:{} 172.20.0.1/32:{}]. Updating annotations. I0327 15:03:48.783595 3944 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.80/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-80.ec2.internal","mac-address":"12:78:66:10:09:29","ip-addresses":["10.0.0.80/24"],"ip-address":"10.0.0.80/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:52e1306a-bd92-4ed7-8b03-3cc58b56001f k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.80/24"}] on node ip-10-0-0-80.ec2.internal I0327 15:03:48.798461 3944 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-80.ec2.internal in network "default" I0327 15:03:48.798462 3944 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:03:48.798632 3944 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 172.371µs I0327 15:03:48.798464 3944 master.go:532] Adding or Updating local node "ip-10-0-0-80.ec2.internal" for network "default" I0327 15:03:48.798726 3944 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.0.80 I0327 15:03:48.798481 3944 egressfirewall.go:628] Syncing node "ip-10-0-0-80.ec2.internal" for egress firewall I0327 15:03:48.798616 3944 node_tracker.go:169] Node ip-10-0-0-80.ec2.internal switch + router changed, syncing services in network "default" I0327 15:03:48.798776 3944 services_controller.go:574] Full service sync requested for network=default I0327 15:03:48.798835 3944 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-80.ec2.internal W0327 15:03:48.799093 3944 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:03:48.803497 3944 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-80.ec2.internal. Updating no re-route policies I0327 15:03:48.803584 3944 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-f7689 I0327 15:03:48.803600 3944 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-f7689 I0327 15:03:48.803626 3944 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-f7689 after 0 failed attempt(s) I0327 15:03:48.803633 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-5z7l5 I0327 15:03:48.803641 3944 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-80.ec2.internal I0327 15:03:48.803651 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-5z7l5 I0327 15:03:48.803654 3944 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-80.ec2.internal I0327 15:03:48.803653 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-dctpb I0327 15:03:48.803655 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-9q9gv I0327 15:03:48.803680 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-dctpb I0327 15:03:48.803682 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-9q9gv I0327 15:03:48.803685 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-ccdzt I0327 15:03:48.803699 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-dctpb after 0 failed attempt(s) I0327 15:03:48.803701 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-ccdzt I0327 15:03:48.803700 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-5d66k I0327 15:03:48.803703 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-5wwpm I0327 15:03:48.803708 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-4hqj8 I0327 15:03:48.803712 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-5d66k I0327 15:03:48.803712 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-80.ec2.internal I0327 15:03:48.803712 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-5wwpm I0327 15:03:48.803717 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-4hqj8 I0327 15:03:48.803719 3944 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-9q9gv] creating logical port openshift-network-diagnostics_network-check-target-9q9gv for pod on switch ip-10-0-0-80.ec2.internal I0327 15:03:48.803721 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-80.ec2.internal I0327 15:03:48.803724 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-ccdzt after 0 failed attempt(s) I0327 15:03:48.803724 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-tn44d I0327 15:03:48.803687 3944 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-80.ec2.internal after 0 failed attempt(s) I0327 15:03:48.803645 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-8xc8g I0327 15:03:48.803737 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-4hqj8 after 0 failed attempt(s) I0327 15:03:48.803740 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-tn44d I0327 15:03:48.803742 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-8xc8g I0327 15:03:48.803688 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-5z7l5 after 0 failed attempt(s) I0327 15:03:48.803739 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-80.ec2.internal after 0 failed attempt(s) I0327 15:03:48.803755 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-8xc8g after 0 failed attempt(s) I0327 15:03:48.803728 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-5d66k after 0 failed attempt(s) I0327 15:03:48.803710 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-ktn4j I0327 15:03:48.803764 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-tn44d after 0 failed attempt(s) I0327 15:03:48.803769 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-ktn4j I0327 15:03:48.803728 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-5wwpm after 0 failed attempt(s) I0327 15:03:48.803819 3944 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-ktn4j] creating logical port openshift-multus_network-metrics-daemon-ktn4j for pod on switch ip-10-0-0-80.ec2.internal I0327 15:03:48.804434 3944 pods.go:271] [openshift-multus/network-metrics-daemon-ktn4j] addLogicalPort took 624.411µs, libovsdb time 274.015µs I0327 15:03:48.804445 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-ktn4j after 0 failed attempt(s) I0327 15:03:48.804524 3944 pods.go:271] [openshift-network-diagnostics/network-check-target-9q9gv] addLogicalPort took 818.232µs, libovsdb time 324.761µs I0327 15:03:48.804532 3944 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-9q9gv after 0 failed attempt(s) I0327 15:03:48.813356 3944 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0327 15:03:48.824385 3944 iptables.go:108] Creating table: mangle chain: OUTPUT I0327 15:03:48.827530 3944 iptables.go:108] Creating table: nat chain: OUTPUT I0327 15:03:48.857118 3944 iptables.go:108] Creating table: nat chain: PREROUTING I0327 15:03:48.928562 3944 iptables.go:108] Creating table: filter chain: FORWARD I0327 15:03:48.945417 3944 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0327 15:03:48.945522 3944 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0327 15:03:48.945563 3944 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0327 15:03:48.945634 3944 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0327 15:03:48.945702 3944 gateway_init.go:325] Gateway and management port readiness took 2.702459681s I0327 15:03:48.945730 3944 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0327 15:03:48.945797 3944 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0327 15:03:48.945810 3944 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0327 15:03:48.945819 3944 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0327 15:03:48.945876 3944 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0327 15:03:48.945956 3944 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 53.911µs I0327 15:03:48.946217 3944 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0327 15:03:48.946241 3944 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.734µs I0327 15:03:48.946249 3944 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 32.306µs I0327 15:03:48.946255 3944 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0327 15:03:48.946263 3944 nad_controller.go:173] [node-nad-controller NAD controller]: started I0327 15:03:48.946268 3944 default_node_network_controller.go:898] Starting the default node network controller I0327 15:03:48.946273 3944 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0327 15:03:48.946352 3944 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 13.212µs I0327 15:03:48.946440 3944 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0327 15:03:48.946448 3944 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0327 15:03:48.946471 3944 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a9d5a0 0xc0002ee070 0xc0002ee150 0xc0002ee1c0 0xc0002ee230 0xc0002ee2a0 0xc0002ee310 0xc000aab9f0 0xc0002ee380 0xc0002ee3f0 0xc0007e8370 0xc0064951f0 0xc0002ee460 map[0x2f67040:0xc00250bce0 0x2f67580:0xc00250be60 0x2f68000:0xc002211e60 0x2f682a0:0xc00250bc80 0x2f687e0:0xc00250bd40 0x2f68a80:0xc00250bda0 0x2f68d20:0xc00250be00 0x2f69500:0xc00250bec0 0x2f697a0:0xc00250bf20 0x2f93d40:0xc0011239e0 0x2f940e0:0xc001123a40 0x2f94bc0:0xc000b9d7a0 0x2f96520:0xc001be6000 0x2f97000:0xc002211d40 0x2f97e80:0xc001be6060] 0xc000aa0d90 0} I0327 15:03:48.946532 3944 gateway.go:151] Starting gateway service sync I0327 15:03:48.947009 3944 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0327 15:03:48.949133 3944 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0327 15:03:48.966976 3944 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0327 15:03:48.968178 3944 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0327 15:03:48.969348 3944 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0327 15:03:49.004382 3944 gateway.go:168] Gateway service sync done. Time taken: 57.836466ms I0327 15:03:49.005037 3944 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32644 I0327 15:03:49.005226 3944 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a9d5a0 0xc0002ee070 0xc0002ee150 0xc0002ee1c0 0xc0002ee230 0xc0002ee2a0 0xc0002ee310 0xc000aab9f0 0xc0002ee380 0xc0002ee3f0 0xc0007e8370 0xc0064951f0 0xc0002ee460 map[0x2f67040:0xc00250bce0 0x2f67580:0xc00250be60 0x2f68000:0xc002211e60 0x2f682a0:0xc00250bc80 0x2f687e0:0xc00250bd40 0x2f68a80:0xc00250bda0 0x2f68d20:0xc00250be00 0x2f69500:0xc00250bec0 0x2f697a0:0xc00250bf20 0x2f93d40:0xc0011239e0 0x2f940e0:0xc001123a40 0x2f94bc0:0xc000b9d7a0 0x2f96520:0xc001be6000 0x2f97000:0xc002211d40 0x2f97e80:0xc001be6060] 0xc000aa0d90 0} I0327 15:03:49.005716 3944 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a9d5a0 0xc0002ee070 0xc0002ee150 0xc0002ee1c0 0xc0002ee230 0xc0002ee2a0 0xc0002ee310 0xc000aab9f0 0xc0002ee380 0xc0002ee3f0 0xc0007e8370 0xc0064951f0 0xc0002ee460 map[0x2f67040:0xc00250bce0 0x2f67580:0xc00250be60 0x2f68000:0xc002211e60 0x2f682a0:0xc00250bc80 0x2f687e0:0xc00250bd40 0x2f68a80:0xc00250bda0 0x2f68d20:0xc00250be00 0x2f69500:0xc00250bec0 0x2f697a0:0xc00250bf20 0x2f93d40:0xc0011239e0 0x2f940e0:0xc001123a40 0x2f94bc0:0xc000b9d7a0 0x2f96520:0xc001be6000 0x2f97000:0xc002211d40 0x2f97e80:0xc001be6060] 0xc000aa0d90 0} I0327 15:03:49.005761 3944 gateway.go:519] Reconciling gateway with updates I0327 15:03:49.011282 3944 gateway.go:326] Spawning Conntrack Rule Check Thread I0327 15:03:49.011370 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:03:49.011386 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:03:49.011393 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:03:49.011400 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:03:49.011409 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:03:49.011411 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 15:03:49.011420 3944 default_node_network_controller.go:943] Gateway and management port readiness took 64.961503ms I0327 15:03:49.011417 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:03:49.011427 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 15:03:49.011431 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:03:49.011454 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:03:49.011452 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:03:49.011462 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 15:03:49.011469 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:03:49.011469 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:03:49.011476 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:03:49.011398 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:03:49.011484 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:03:49.011488 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:03:49.011492 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:03:49.011492 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:03:49.011497 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:03:49.011499 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 15:03:49.011499 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 15:03:49.011502 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 15:03:49.011498 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:03:49.011512 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:03:49.011515 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 15:03:49.011518 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 15:03:49.011521 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 15:03:49.011525 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 15:03:49.011436 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:03:49.011530 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:49.011536 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 15:03:49.011537 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 15:03:49.011541 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:03:49.011547 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:49.011545 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 15:03:49.011554 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:03:49.011506 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:03:49.011559 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 15:03:49.011562 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:03:49.011567 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:03:49.011573 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:03:49.011573 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 15:03:49.011576 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:03:49.011455 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:03:49.011584 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:03:49.011595 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:03:49.011555 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:03:49.011601 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:03:49.011602 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:03:49.011482 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:03:49.011614 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:03:49.011620 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:03:49.011623 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:03:49.011564 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:03:49.011628 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:03:49.011634 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:03:49.011482 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 15:03:49.011636 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:03:49.011641 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:03:49.011645 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:03:49.011640 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 15:03:49.011648 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:03:49.011479 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 15:03:49.011611 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:03:49.011660 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 15:03:49.011625 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 15:03:49.011608 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 15:03:49.011614 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 15:03:49.011694 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 15:03:49.011706 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 15:03:49.011622 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:03:49.011717 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:03:49.011371 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:03:49.011731 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:03:49.011734 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:03:49.011743 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:03:49.011744 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 15:03:49.011628 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:03:49.011758 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:03:49.011764 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 15:03:49.011634 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:03:49.011801 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:03:49.011608 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:03:49.011706 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 15:03:49.011829 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:03:49.011600 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:03:49.011652 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 15:03:49.011659 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:03:49.011862 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 15:03:49.011579 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:03:49.011881 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:03:49.011887 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:03:49.011893 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:03:49.011641 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:03:49.011913 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:03:49.011922 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:03:49.011927 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:03:49.011872 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 15:03:49.012032 3944 port_claim.go:71] Svc openshift-ingress/router-default: port 30374 is already open, no action needed I0327 15:03:49.012048 3944 port_claim.go:71] Svc openshift-ingress/router-default: port 31767 is already open, no action needed I0327 15:03:49.011632 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:03:49.012083 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:03:49.012094 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 15:03:49.011906 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:03:49.012104 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:03:49.012551 3944 node_ip_handler_linux.go:160] Node IP manager is running W0327 15:03:49.048324 3944 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 15:03:49.048501 3944 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.130.0.1 on link ovn-k8s-mp0 with MTU: 0 W0327 15:03:49.048707 3944 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:82:00:01/10.130.0.1) on link ovn-k8s-mp0 I0327 15:03:49.058083 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 15:03:49.058107 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 15:03:49.058115 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 15:03:49.058135 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 15:03:49.058174 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 15:03:49.058189 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 15:03:49.058204 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 15:03:49.058219 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 15:03:49.097311 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 15:03:49.097345 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 15:03:49.097354 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 15:03:49.097360 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 15:03:49.097370 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 15:03:49.097378 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 15:03:49.097384 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 15:03:49.097390 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 15:03:49.097396 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 15:03:49.097402 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 15:03:49.097408 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 15:03:49.097413 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 15:03:49.097420 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 15:03:49.097426 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 15:03:49.097435 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 15:03:49.097441 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 15:03:49.097446 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 15:03:49.137707 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 15:03:49.137729 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 15:03:50.634744 3944 default_node_network_controller.go:1364] Starting node controller node sync W0327 15:03:50.634833 3944 egressip_healthcheck.go:76] Health checking using insecure connection I0327 15:03:50.634880 3944 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.130.0.2:9107 I0327 15:03:50.646544 3944 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.776549ms I0327 15:03:50.646702 3944 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-39.ec2.internal", IP: 10.0.2.39 to PMTUD blocking rules I0327 15:03:50.646714 3944 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-39.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:03:50.646701 3944 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-176.ec2.internal", IP: 10.0.1.176 to PMTUD blocking rules I0327 15:03:50.646817 3944 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-176.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0327 15:03:50.658074 3944 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0327 15:03:50.658292 3944 egressservice_node.go:93] Setting up event handlers for Egress Services I0327 15:03:50.658431 3944 egressservice_node.go:183] Starting Egress Services Controller I0327 15:03:50.658448 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0327 15:03:50.658457 3944 shared_informer.go:356] "Caches are synced" controller="egressservices" I0327 15:03:50.658463 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0327 15:03:50.658468 3944 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0327 15:03:50.658486 3944 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0327 15:03:50.658493 3944 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0327 15:03:50.658498 3944 egressservice_node.go:197] Repairing Egress Services I0327 15:03:50.668322 3944 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0327 15:03:50.668333 3944 external_controller.go:278] Starting Admin Policy Based Route Controller I0327 15:03:50.668502 3944 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0327 15:03:50.668510 3944 link_network_manager.go:117] Link manager is running I0327 15:03:50.668540 3944 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0327 15:03:50.668641 3944 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0327 15:03:50.669230 3944 vrf_manager.go:128] VRF manager is running I0327 15:03:50.669501 3944 node_controller_manager.go:437] Removing flows to drop GARP I0327 15:03:50.669510 3944 gateway.go:519] Reconciling gateway with updates I0327 15:03:50.693621 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0327 15:03:50.693639 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:03:50.693645 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:03:50.693648 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:03:50.693661 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:03:50.693664 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0327 15:03:50.693664 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0327 15:03:50.693685 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0327 15:03:50.693687 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:03:50.693628 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:03:50.693703 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0327 15:03:50.693692 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0327 15:03:50.693708 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:03:50.693712 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:03:50.693723 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0327 15:03:50.693726 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0327 15:03:50.693729 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0327 15:03:50.693730 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:03:50.693626 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0327 15:03:50.693740 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:03:50.693743 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0327 15:03:50.693745 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:03:50.693747 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0327 15:03:50.693653 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0327 15:03:50.693760 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:03:50.693766 3944 port_claim.go:71] Svc openshift-ingress/router-default: port 30374 is already open, no action needed I0327 15:03:50.693768 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:03:50.693773 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0327 15:03:50.693774 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0327 15:03:50.693750 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:03:50.693783 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:03:50.693788 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0327 15:03:50.693792 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0327 15:03:50.693795 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:03:50.693744 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0327 15:03:50.693807 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:03:50.693818 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:03:50.693732 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0327 15:03:50.693756 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0327 15:03:50.693831 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0327 15:03:50.693831 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0327 15:03:50.693813 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:03:50.693839 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:03:50.693840 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0327 15:03:50.693849 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:03:50.693772 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0327 15:03:50.693859 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:03:50.693805 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0327 15:03:50.693855 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0327 15:03:50.693869 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0327 15:03:50.693872 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:03:50.693877 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:03:50.693877 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0327 15:03:50.693882 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0327 15:03:50.693885 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0327 15:03:50.693886 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0327 15:03:50.693816 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0327 15:03:50.693893 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0327 15:03:50.693897 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0327 15:03:50.693865 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0327 15:03:50.693900 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:03:50.693775 3944 port_claim.go:71] Svc openshift-ingress/router-default: port 31767 is already open, no action needed I0327 15:03:50.693912 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:03:50.693917 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0327 15:03:50.693924 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0327 15:03:50.693850 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0327 15:03:50.693928 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0327 15:03:50.693933 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0327 15:03:50.693938 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0327 15:03:50.693746 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:03:50.693947 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0327 15:03:50.693950 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0327 15:03:50.693953 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0327 15:03:50.693956 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0327 15:03:50.693959 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0327 15:03:50.693835 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0327 15:03:50.693970 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0327 15:03:50.693883 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:03:50.693975 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:03:50.693982 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:03:50.693989 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:03:50.693995 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0327 15:03:50.693705 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0327 15:03:50.694001 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0327 15:03:50.693956 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0327 15:03:50.693945 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:03:50.694027 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0327 15:03:50.693874 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:03:50.693756 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0327 15:03:50.693909 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0327 15:03:50.693863 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:50.693985 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0327 15:03:50.694063 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0327 15:03:50.693974 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:03:50.693983 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0327 15:03:50.694000 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0327 15:03:50.693997 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0327 15:03:50.694114 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0327 15:03:50.694047 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0327 15:03:50.693958 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0327 15:03:50.694069 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0327 15:03:50.693887 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:03:50.694183 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0327 15:03:50.694079 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0327 15:03:50.694005 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0327 15:03:50.694294 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0327 15:03:50.694431 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:03:50.694443 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0327 15:03:50.694457 3944 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:03:50.694469 3944 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0327 15:03:50.694473 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0327 15:03:50.732080 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0327 15:03:50.732094 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0327 15:03:50.732111 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0327 15:03:50.732118 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0327 15:03:50.732124 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0327 15:03:50.732132 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0327 15:03:50.732139 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0327 15:03:50.732146 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0327 15:03:50.732163 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0327 15:03:50.732184 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0327 15:03:50.732192 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0327 15:03:50.732198 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0327 15:03:50.732204 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0327 15:03:50.769867 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0327 15:03:50.788817 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0327 15:03:50.788824 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0327 15:03:50.788834 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0327 15:03:50.788843 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0327 15:03:50.788856 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0327 15:03:50.788865 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0327 15:03:50.788882 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0327 15:03:50.788890 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0327 15:03:50.788896 3944 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0327 15:03:52.243802 3944 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:03:52.243835 3944 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 40.204µs I0327 15:03:52.267624 3944 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-66db7c9c95-jxct4] creating logical port openshift-image-registry_image-registry-66db7c9c95-jxct4 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:03:52.267809 3944 kube.go:256] Updating pod openshift-image-registry/image-registry-66db7c9c95-jxct4 I0327 15:03:52.288131 3944 pod.go:62] [openshift-image-registry/image-registry-66db7c9c95-jxct4] pod update took 20.368465ms I0327 15:03:52.288153 3944 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-66db7c9c95-jxct4] addLogicalPort annotation time took 20.395537ms I0327 15:03:52.289401 3944 pods.go:271] [openshift-image-registry/image-registry-66db7c9c95-jxct4] addLogicalPort took 21.796324ms, libovsdb time 984.141µs I0327 15:03:52.296159 3944 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-q6nvm] creating logical port openshift-ingress-canary_ingress-canary-q6nvm for pod on switch ip-10-0-0-80.ec2.internal I0327 15:03:52.296251 3944 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-q6nvm I0327 15:03:52.302204 3944 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-fnjs8] creating logical port openshift-dns_dns-default-fnjs8 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:03:52.302314 3944 kube.go:256] Updating pod openshift-dns/dns-default-fnjs8 I0327 15:03:52.315067 3944 pod.go:62] [openshift-ingress-canary/ingress-canary-q6nvm] pod update took 18.829417ms I0327 15:03:52.315093 3944 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-q6nvm] addLogicalPort annotation time took 18.862452ms I0327 15:03:52.316263 3944 pods.go:271] [openshift-ingress-canary/ingress-canary-q6nvm] addLogicalPort took 20.162516ms, libovsdb time 903.797µs I0327 15:03:52.318551 3944 pod.go:62] [openshift-dns/dns-default-fnjs8] pod update took 16.265949ms I0327 15:03:52.318574 3944 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-fnjs8] addLogicalPort annotation time took 16.29211ms I0327 15:03:52.319543 3944 pods.go:271] [openshift-dns/dns-default-fnjs8] addLogicalPort took 17.354433ms, libovsdb time 679.533µs I0327 15:03:52.694151 3944 cni.go:403] [openshift-ingress-canary/ingress-canary-q6nvm 1f086ebfaf22bbd7661b14bf23bc59263e30695b8f8150ffb48863a7858f7915 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-q6nvm 1f086ebfaf22bbd7661b14bf23bc59263e30695b8f8150ffb48863a7858f7915 network default NAD default] I0327 15:03:52.732624 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-q6nvm, hostIfaceName: 1f086ebfaf22bbd, network: default, NAD default, SandboxID: "1f086ebfaf22bbd7661b14bf23bc59263e30695b8f8150ffb48863a7858f7915", PCI device ID: , UID: "ba960992-deda-4bd6-a931-f733db43cf25", MAC: 0a:58:0a:82:00:06, IPs: [10.130.0.6/23] I0327 15:03:52.796469 3944 cni.go:424] [openshift-ingress-canary/ingress-canary-q6nvm 1f086ebfaf22bbd7661b14bf23bc59263e30695b8f8150ffb48863a7858f7915 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-q6nvm 1f086ebfaf22bbd7661b14bf23bc59263e30695b8f8150ffb48863a7858f7915 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:58:e2:3f:f9:2a\",\"name\":\"1f086ebfaf22bbd\"},{\"mac\":\"0a:58:0a:82:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/40438c99-3ca5-42c5-90e2-5288e99e8d44\"}],\"ips\":[{\"address\":\"10.130.0.6/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:04:24.056399 3944 cni.go:403] [openshift-network-diagnostics/network-check-target-9q9gv 7fc9b1269d04710990cdc448a7110d9aab2e99096e6aef44beed2aba860b6057 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-9q9gv 7fc9b1269d04710990cdc448a7110d9aab2e99096e6aef44beed2aba860b6057 network default NAD default] I0327 15:04:24.064538 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-9q9gv, hostIfaceName: 7fc9b1269d04710, network: default, NAD default, SandboxID: "7fc9b1269d04710990cdc448a7110d9aab2e99096e6aef44beed2aba860b6057", PCI device ID: , UID: "43d09da6-9859-469f-9e38-0d541847ae8e", MAC: 0a:58:0a:82:00:03, IPs: [10.130.0.3/23] I0327 15:04:24.106062 3944 cni.go:424] [openshift-network-diagnostics/network-check-target-9q9gv 7fc9b1269d04710990cdc448a7110d9aab2e99096e6aef44beed2aba860b6057 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-9q9gv 7fc9b1269d04710990cdc448a7110d9aab2e99096e6aef44beed2aba860b6057 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:a6:61:46:d1:dc\",\"name\":\"7fc9b1269d04710\"},{\"mac\":\"0a:58:0a:82:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cfd6e2cb-3839-494e-9f94-112cbb689fd0\"}],\"ips\":[{\"address\":\"10.130.0.3/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:04:32.274008 3944 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5f5595559-2xf5k] creating logical port openshift-ingress_router-default-5f5595559-2xf5k for pod on switch ip-10-0-0-80.ec2.internal I0327 15:04:32.274125 3944 kube.go:256] Updating pod openshift-ingress/router-default-5f5595559-2xf5k I0327 15:04:32.295070 3944 pod.go:62] [openshift-ingress/router-default-5f5595559-2xf5k] pod update took 20.974125ms I0327 15:04:32.295091 3944 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-5f5595559-2xf5k] addLogicalPort annotation time took 20.999886ms I0327 15:04:32.296274 3944 pods.go:271] [openshift-ingress/router-default-5f5595559-2xf5k] addLogicalPort took 22.284967ms, libovsdb time 941.266µs W0327 15:04:58.242965 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0327 15:05:48.528398 3944 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:05:48.528969 3944 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:05:48.529001 3944 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:05:48.529014 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0327 15:05:48.529025 3944 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:05:48.529030 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0327 15:05:49.551525 3944 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:05:49.551548 3944 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:05:49.551562 3944 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:05:49.551581 3944 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390425fq of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:05:53.978455 3944 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 15:05:53.988875 3944 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0327 15:06:58.413908 3944 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-65fb97d8cf-4dvvk] creating logical port openshift-ingress_router-default-65fb97d8cf-4dvvk for pod on switch ip-10-0-0-80.ec2.internal I0327 15:06:58.414064 3944 kube.go:256] Updating pod openshift-ingress/router-default-65fb97d8cf-4dvvk I0327 15:06:58.444108 3944 pod.go:62] [openshift-ingress/router-default-65fb97d8cf-4dvvk] pod update took 30.07693ms I0327 15:06:58.444132 3944 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-65fb97d8cf-4dvvk] addLogicalPort annotation time took 30.106358ms I0327 15:06:58.445206 3944 pods.go:271] [openshift-ingress/router-default-65fb97d8cf-4dvvk] addLogicalPort took 31.319141ms, libovsdb time 779.694µs I0327 15:06:58.498197 3944 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-zss7x] creating logical port openshift-service-ca-operator_service-ca-operator-7b89bc695f-zss7x for pod on switch ip-10-0-0-80.ec2.internal I0327 15:06:58.498364 3944 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-7b89bc695f-zss7x I0327 15:06:58.529292 3944 pod.go:62] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-zss7x] pod update took 30.955295ms I0327 15:06:58.529316 3944 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-zss7x] addLogicalPort annotation time took 30.985106ms I0327 15:06:58.530419 3944 pods.go:271] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-zss7x] addLogicalPort took 32.242487ms, libovsdb time 839.965µs I0327 15:06:58.849272 3944 cni.go:403] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-zss7x 55530d0bb60cbf38115c42a43459fc15b7a5e04802b5803138b17bf9f7335422 network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-zss7x 55530d0bb60cbf38115c42a43459fc15b7a5e04802b5803138b17bf9f7335422 network default NAD default] I0327 15:06:58.857542 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-7b89bc695f-zss7x, hostIfaceName: 55530d0bb60cbf3, network: default, NAD default, SandboxID: "55530d0bb60cbf38115c42a43459fc15b7a5e04802b5803138b17bf9f7335422", PCI device ID: , UID: "5e47eca2-c097-47a7-b8b1-a7c99c4a4688", MAC: 0a:58:0a:82:00:0a, IPs: [10.130.0.10/23] I0327 15:06:58.898772 3944 cni.go:424] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-zss7x 55530d0bb60cbf38115c42a43459fc15b7a5e04802b5803138b17bf9f7335422 network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-zss7x 55530d0bb60cbf38115c42a43459fc15b7a5e04802b5803138b17bf9f7335422 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:28:b2:a4:c5:7d\",\"name\":\"55530d0bb60cbf3\"},{\"mac\":\"0a:58:0a:82:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0e1377cf-fe21-4199-828c-1970a7c65bf6\"}],\"ips\":[{\"address\":\"10.130.0.10/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:07:02.628229 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 15:07:03.535300 3944 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0327 15:07:03.536277 3944 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 944.797µs I0327 15:07:03.557923 3944 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0327 15:07:04.319376 3944 namespace.go:98] [openshift-service-ca] adding namespace I0327 15:07:04.320365 3944 namespace.go:102] [openshift-service-ca] adding namespace took 959.179µs I0327 15:07:04.339000 3944 namespace.go:142] [openshift-service-ca] updating namespace I0327 15:07:05.986511 3944 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-f59fc589b-d6587] creating logical port openshift-service-ca_service-ca-f59fc589b-d6587 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:05.986606 3944 kube.go:256] Updating pod openshift-service-ca/service-ca-f59fc589b-d6587 I0327 15:07:05.998397 3944 pod.go:62] [openshift-service-ca/service-ca-f59fc589b-d6587] pod update took 11.815798ms I0327 15:07:05.998425 3944 base_network_controller_pods.go:937] [default/openshift-service-ca/service-ca-f59fc589b-d6587] addLogicalPort annotation time took 11.848243ms I0327 15:07:05.999651 3944 pods.go:271] [openshift-service-ca/service-ca-f59fc589b-d6587] addLogicalPort took 13.158812ms, libovsdb time 850.049µs I0327 15:07:06.641800 3944 cni.go:403] [openshift-service-ca/service-ca-f59fc589b-d6587 7afaaff2a288f74a418e52580f1145aafb4830f6d857d8b6ceb1d6b415f2ca25 network default NAD default] ADD starting CNI request [openshift-service-ca/service-ca-f59fc589b-d6587 7afaaff2a288f74a418e52580f1145aafb4830f6d857d8b6ceb1d6b415f2ca25 network default NAD default] I0327 15:07:06.650807 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-f59fc589b-d6587, hostIfaceName: 7afaaff2a288f74, network: default, NAD default, SandboxID: "7afaaff2a288f74a418e52580f1145aafb4830f6d857d8b6ceb1d6b415f2ca25", PCI device ID: , UID: "1620a879-6604-495f-98cd-827da889d48e", MAC: 0a:58:0a:82:00:0b, IPs: [10.130.0.11/23] I0327 15:07:06.692152 3944 cni.go:424] [openshift-service-ca/service-ca-f59fc589b-d6587 7afaaff2a288f74a418e52580f1145aafb4830f6d857d8b6ceb1d6b415f2ca25 network default NAD default] ADD finished CNI request [openshift-service-ca/service-ca-f59fc589b-d6587 7afaaff2a288f74a418e52580f1145aafb4830f6d857d8b6ceb1d6b415f2ca25 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:42:2f:5f:2e:e4\",\"name\":\"7afaaff2a288f74\"},{\"mac\":\"0a:58:0a:82:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c411788b-c2b0-49b8-9fe1-97ada3a780dd\"}],\"ips\":[{\"address\":\"10.130.0.11/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:07:10.593405 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0327 15:07:12.673261 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5877c497c6-7qwf2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:12.673876 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5877c497c6-7qwf2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:12.684048 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5877c497c6-7qwf2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:12.684069 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5877c497c6-7qwf2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:12.691702 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5877c497c6-7qwf2 I0327 15:07:12.691718 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5877c497c6-7qwf2 I0327 15:07:13.617121 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-66db7c9c95-jxct4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:13.617173 3944 pods.go:176] Deleting pod: openshift-image-registry/image-registry-66db7c9c95-jxct4 I0327 15:07:13.618867 3944 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-66db7c9c95-jxct4, ips: 10.130.0.5 I0327 15:07:13.618901 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-66db7c9c95-jxct4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:13.625088 3944 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-65c4b7d4b8-78rvv] creating logical port openshift-image-registry_image-registry-65c4b7d4b8-78rvv for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:13.625171 3944 kube.go:256] Updating pod openshift-image-registry/image-registry-65c4b7d4b8-78rvv I0327 15:07:13.642108 3944 pod.go:62] [openshift-image-registry/image-registry-65c4b7d4b8-78rvv] pod update took 16.96128ms I0327 15:07:13.642130 3944 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-65c4b7d4b8-78rvv] addLogicalPort annotation time took 16.989007ms I0327 15:07:13.643313 3944 pods.go:271] [openshift-image-registry/image-registry-65c4b7d4b8-78rvv] addLogicalPort took 18.237159ms, libovsdb time 885.14µs I0327 15:07:13.643363 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-66db7c9c95-jxct4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:13.643374 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-66db7c9c95-jxct4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:13.643386 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-66db7c9c95-jxct4 I0327 15:07:13.643393 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-66db7c9c95-jxct4 W0327 15:07:13.679146 3944 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:07:13.984050 3944 cni.go:403] [openshift-image-registry/image-registry-65c4b7d4b8-78rvv f4bcf45e02047d70caf5ec59989ab9dabf0271f51967882955bcfb1f2defd0b1 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-65c4b7d4b8-78rvv f4bcf45e02047d70caf5ec59989ab9dabf0271f51967882955bcfb1f2defd0b1 network default NAD default] I0327 15:07:13.993613 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-65c4b7d4b8-78rvv, hostIfaceName: f4bcf45e02047d7, network: default, NAD default, SandboxID: "f4bcf45e02047d70caf5ec59989ab9dabf0271f51967882955bcfb1f2defd0b1", PCI device ID: , UID: "a8ef6b17-a791-40f8-9637-ef20aece51b2", MAC: 0a:58:0a:82:00:0c, IPs: [10.130.0.12/23] I0327 15:07:14.037086 3944 cni.go:424] [openshift-image-registry/image-registry-65c4b7d4b8-78rvv f4bcf45e02047d70caf5ec59989ab9dabf0271f51967882955bcfb1f2defd0b1 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-65c4b7d4b8-78rvv f4bcf45e02047d70caf5ec59989ab9dabf0271f51967882955bcfb1f2defd0b1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:a3:38:7b:c0:1e\",\"name\":\"f4bcf45e02047d7\"},{\"mac\":\"0a:58:0a:82:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3a39f91f-93e7-4b0e-8d60-6a5e21a3c48c\"}],\"ips\":[{\"address\":\"10.130.0.12/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:07:30.045606 3944 cni.go:403] [openshift-multus/network-metrics-daemon-ktn4j 92cba2184e25acfe554fa219095fe6977e0cbbd518fa52a8580d1c41ff12772b network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-ktn4j 92cba2184e25acfe554fa219095fe6977e0cbbd518fa52a8580d1c41ff12772b network default NAD default] I0327 15:07:30.054925 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-ktn4j, hostIfaceName: 92cba2184e25acf, network: default, NAD default, SandboxID: "92cba2184e25acfe554fa219095fe6977e0cbbd518fa52a8580d1c41ff12772b", PCI device ID: , UID: "41175c56-6767-46d9-856d-3682b6eed5b4", MAC: 0a:58:0a:82:00:04, IPs: [10.130.0.4/23] I0327 15:07:30.096937 3944 cni.go:424] [openshift-multus/network-metrics-daemon-ktn4j 92cba2184e25acfe554fa219095fe6977e0cbbd518fa52a8580d1c41ff12772b network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-ktn4j 92cba2184e25acfe554fa219095fe6977e0cbbd518fa52a8580d1c41ff12772b network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:8d:e0:f0:4e:79\",\"name\":\"92cba2184e25acf\"},{\"mac\":\"0a:58:0a:82:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/18100d9b-dd4e-443c-b6ed-cbc812d75f19\"}],\"ips\":[{\"address\":\"10.130.0.4/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:07:30.578447 3944 cni.go:403] [openshift-ingress/router-default-65fb97d8cf-4dvvk 3141544da5bcc9b5327173c5ad1096b4dc2ac65b2d1ec9ed3b40651d4cb005c8 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-65fb97d8cf-4dvvk 3141544da5bcc9b5327173c5ad1096b4dc2ac65b2d1ec9ed3b40651d4cb005c8 network default NAD default] I0327 15:07:30.589093 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-65fb97d8cf-4dvvk, hostIfaceName: 3141544da5bcc9b, network: default, NAD default, SandboxID: "3141544da5bcc9b5327173c5ad1096b4dc2ac65b2d1ec9ed3b40651d4cb005c8", PCI device ID: , UID: "f539efd7-9ed3-4686-a7a3-8b4736151ba9", MAC: 0a:58:0a:82:00:09, IPs: [10.130.0.9/23] I0327 15:07:30.630198 3944 cni.go:424] [openshift-ingress/router-default-65fb97d8cf-4dvvk 3141544da5bcc9b5327173c5ad1096b4dc2ac65b2d1ec9ed3b40651d4cb005c8 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-65fb97d8cf-4dvvk 3141544da5bcc9b5327173c5ad1096b4dc2ac65b2d1ec9ed3b40651d4cb005c8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"62:21:51:f2:e1:ed\",\"name\":\"3141544da5bcc9b\"},{\"mac\":\"0a:58:0a:82:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46eeed77-df90-45d9-8092-749c51e4acc5\"}],\"ips\":[{\"address\":\"10.130.0.9/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:07:32.068162 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0327 15:07:32.068180 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0327 15:07:32.071812 3944 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:07:32.076523 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:07:32.076543 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:07:32.076549 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:07:32.080144 3944 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0327 15:07:32.080159 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0327 15:07:32.080163 3944 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0327 15:07:32.193144 3944 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-g6smd] creating logical port openshift-insights_insights-runtime-extractor-g6smd for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:32.193236 3944 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-g6smd I0327 15:07:32.221191 3944 pod.go:62] [openshift-insights/insights-runtime-extractor-g6smd] pod update took 27.986047ms I0327 15:07:32.221223 3944 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-g6smd] addLogicalPort annotation time took 28.020344ms I0327 15:07:32.222216 3944 pods.go:271] [openshift-insights/insights-runtime-extractor-g6smd] addLogicalPort took 29.08947ms, libovsdb time 686.362µs W0327 15:07:32.258844 3944 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:07:32.287467 3944 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-67sv9] creating logical port openshift-console_downloads-56d45c498b-67sv9 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:32.287554 3944 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-67sv9 W0327 15:07:32.291264 3944 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0327 15:07:32.311762 3944 pod.go:62] [openshift-console/downloads-56d45c498b-67sv9] pod update took 24.22674ms I0327 15:07:32.311787 3944 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-67sv9] addLogicalPort annotation time took 24.25962ms I0327 15:07:32.312831 3944 pods.go:271] [openshift-console/downloads-56d45c498b-67sv9] addLogicalPort took 25.3827ms, libovsdb time 771.421µs I0327 15:07:32.553635 3944 cni.go:403] [openshift-insights/insights-runtime-extractor-g6smd 84b6e5f017c644c4ce46a72c81eb29dd28da1848c798cc172a4e238267913ce3 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-g6smd 84b6e5f017c644c4ce46a72c81eb29dd28da1848c798cc172a4e238267913ce3 network default NAD default] I0327 15:07:32.562989 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-g6smd, hostIfaceName: 84b6e5f017c644c, network: default, NAD default, SandboxID: "84b6e5f017c644c4ce46a72c81eb29dd28da1848c798cc172a4e238267913ce3", PCI device ID: , UID: "3b1b1613-959d-4915-a7da-5aecfb770df1", MAC: 0a:58:0a:82:00:0d, IPs: [10.130.0.13/23] I0327 15:07:32.604734 3944 cni.go:424] [openshift-insights/insights-runtime-extractor-g6smd 84b6e5f017c644c4ce46a72c81eb29dd28da1848c798cc172a4e238267913ce3 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-g6smd 84b6e5f017c644c4ce46a72c81eb29dd28da1848c798cc172a4e238267913ce3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:e2:b1:94:ea:d1\",\"name\":\"84b6e5f017c644c\"},{\"mac\":\"0a:58:0a:82:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6cc9f029-b8df-4372-ae94-e7461076c27f\"}],\"ips\":[{\"address\":\"10.130.0.13/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:07:32.645695 3944 cni.go:403] [openshift-console/downloads-56d45c498b-67sv9 f339adde53d11942f61ccd359294531d7134ff9fd6d4cf80eb25553ab998f517 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-67sv9 f339adde53d11942f61ccd359294531d7134ff9fd6d4cf80eb25553ab998f517 network default NAD default] I0327 15:07:32.655272 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-67sv9, hostIfaceName: f339adde53d1194, network: default, NAD default, SandboxID: "f339adde53d11942f61ccd359294531d7134ff9fd6d4cf80eb25553ab998f517", PCI device ID: , UID: "64941d1e-a306-4bb7-8a35-87eb0de25c99", MAC: 0a:58:0a:82:00:0e, IPs: [10.130.0.14/23] I0327 15:07:32.697172 3944 cni.go:424] [openshift-console/downloads-56d45c498b-67sv9 f339adde53d11942f61ccd359294531d7134ff9fd6d4cf80eb25553ab998f517 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-67sv9 f339adde53d11942f61ccd359294531d7134ff9fd6d4cf80eb25553ab998f517 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:df:9a:18:11:32\",\"name\":\"f339adde53d1194\"},{\"mac\":\"0a:58:0a:82:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e10951b9-e0f0-4561-bf90-e67b9e3b9db2\"}],\"ips\":[{\"address\":\"10.130.0.14/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:07:32.792310 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0327 15:07:32.792331 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0327 15:07:32.863608 3944 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0327 15:07:32.892589 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:07:32.892610 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0327 15:07:32.892615 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default I0327 15:07:33.102954 3944 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-58hsq] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-58hsq for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:33.103065 3944 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-58hsq I0327 15:07:33.116558 3944 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-58hsq] pod update took 13.527698ms I0327 15:07:33.116585 3944 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-58hsq] addLogicalPort annotation time took 13.560673ms I0327 15:07:33.117796 3944 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-58hsq] addLogicalPort took 14.86008ms, libovsdb time 887.984µs I0327 15:07:33.466021 3944 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-58hsq 10f86dbc225479f9a0d9acf2b172eba15947c5577d1d15085368d00552cd5b18 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-58hsq 10f86dbc225479f9a0d9acf2b172eba15947c5577d1d15085368d00552cd5b18 network default NAD default] I0327 15:07:33.474307 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-58hsq, hostIfaceName: 10f86dbc225479f, network: default, NAD default, SandboxID: "10f86dbc225479f9a0d9acf2b172eba15947c5577d1d15085368d00552cd5b18", PCI device ID: , UID: "417187a5-02c3-4420-8c79-105d39a31cf2", MAC: 0a:58:0a:82:00:0f, IPs: [10.130.0.15/23] I0327 15:07:33.517342 3944 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-58hsq 10f86dbc225479f9a0d9acf2b172eba15947c5577d1d15085368d00552cd5b18 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-58hsq 10f86dbc225479f9a0d9acf2b172eba15947c5577d1d15085368d00552cd5b18 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:0b:c7:77:35:04\",\"name\":\"10f86dbc225479f\"},{\"mac\":\"0a:58:0a:82:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e72110ec-40b7-48c7-9bff-3c793a026148\"}],\"ips\":[{\"address\":\"10.130.0.15/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:07:36.767526 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0327 15:07:36.987701 3944 base_network_controller_pods.go:487] [default/openshift-console/console-8579887566-h47gd] creating logical port openshift-console_console-8579887566-h47gd for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:36.987822 3944 kube.go:256] Updating pod openshift-console/console-8579887566-h47gd I0327 15:07:37.001488 3944 pod.go:62] [openshift-console/console-8579887566-h47gd] pod update took 13.695516ms I0327 15:07:37.001514 3944 base_network_controller_pods.go:937] [default/openshift-console/console-8579887566-h47gd] addLogicalPort annotation time took 13.728402ms I0327 15:07:37.002894 3944 pods.go:271] [openshift-console/console-8579887566-h47gd] addLogicalPort took 15.237725ms, libovsdb time 966.43µs I0327 15:07:37.388133 3944 cni.go:403] [openshift-console/console-8579887566-h47gd 2f10f8f96a32f73f84fa9f9fc546b4c47996456e69ee9ea235b950d389bd38da network default NAD default] ADD starting CNI request [openshift-console/console-8579887566-h47gd 2f10f8f96a32f73f84fa9f9fc546b4c47996456e69ee9ea235b950d389bd38da network default NAD default] I0327 15:07:37.397836 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-8579887566-h47gd, hostIfaceName: 2f10f8f96a32f73, network: default, NAD default, SandboxID: "2f10f8f96a32f73f84fa9f9fc546b4c47996456e69ee9ea235b950d389bd38da", PCI device ID: , UID: "b890aecf-3df2-426d-9740-7e95ab32aab6", MAC: 0a:58:0a:82:00:10, IPs: [10.130.0.16/23] I0327 15:07:37.444135 3944 cni.go:424] [openshift-console/console-8579887566-h47gd 2f10f8f96a32f73f84fa9f9fc546b4c47996456e69ee9ea235b950d389bd38da network default NAD default] ADD finished CNI request [openshift-console/console-8579887566-h47gd 2f10f8f96a32f73f84fa9f9fc546b4c47996456e69ee9ea235b950d389bd38da network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:7e:11:12:4b:cd\",\"name\":\"2f10f8f96a32f73\"},{\"mac\":\"0a:58:0a:82:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/998dbfdf-c2d4-4fc9-a8a1-0d8124dd81ff\"}],\"ips\":[{\"address\":\"10.130.0.16/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:07:37.748019 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0327 15:07:41.705385 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0327 15:07:42.159384 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0327 15:07:43.787166 3944 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-8b2zf] creating logical port openshift-monitoring_kube-state-metrics-664d8f7fc8-8b2zf for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:43.787258 3944 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-664d8f7fc8-8b2zf I0327 15:07:43.856606 3944 pod.go:62] [openshift-monitoring/kube-state-metrics-664d8f7fc8-8b2zf] pod update took 69.370889ms I0327 15:07:43.856635 3944 base_network_controller_pods.go:937] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-8b2zf] addLogicalPort annotation time took 69.403783ms I0327 15:07:43.857801 3944 pods.go:271] [openshift-monitoring/kube-state-metrics-664d8f7fc8-8b2zf] addLogicalPort took 70.647369ms, libovsdb time 788.654µs I0327 15:07:44.142362 3944 cni.go:403] [openshift-monitoring/kube-state-metrics-664d8f7fc8-8b2zf 3375e0d7786aa9d7f0749407f80977a98fc3f59990d97d0d9baf447216e7e746 network default NAD default] ADD starting CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-8b2zf 3375e0d7786aa9d7f0749407f80977a98fc3f59990d97d0d9baf447216e7e746 network default NAD default] I0327 15:07:44.150462 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-664d8f7fc8-8b2zf, hostIfaceName: 3375e0d7786aa9d, network: default, NAD default, SandboxID: "3375e0d7786aa9d7f0749407f80977a98fc3f59990d97d0d9baf447216e7e746", PCI device ID: , UID: "44252065-8ef0-4d07-b1c3-222acef5d7f0", MAC: 0a:58:0a:82:00:11, IPs: [10.130.0.17/23] I0327 15:07:44.194818 3944 cni.go:424] [openshift-monitoring/kube-state-metrics-664d8f7fc8-8b2zf 3375e0d7786aa9d7f0749407f80977a98fc3f59990d97d0d9baf447216e7e746 network default NAD default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-8b2zf 3375e0d7786aa9d7f0749407f80977a98fc3f59990d97d0d9baf447216e7e746 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:75:cb:ec:f8:36\",\"name\":\"3375e0d7786aa9d\"},{\"mac\":\"0a:58:0a:82:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f7312871-7563-4791-a8c4-f65f86f5e88f\"}],\"ips\":[{\"address\":\"10.130.0.17/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:07:45.640523 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0327 15:07:47.197784 3944 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:47.197804 3944 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:47.197809 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:48.193662 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 15:07:48.193706 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 15:07:48.423746 3944 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-jjdwr] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-jjdwr for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:48.423934 3944 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-jjdwr I0327 15:07:48.447137 3944 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jjdwr] pod update took 23.240671ms I0327 15:07:48.447164 3944 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-jjdwr] addLogicalPort annotation time took 23.275529ms I0327 15:07:48.448705 3944 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jjdwr] addLogicalPort took 24.980651ms, libovsdb time 1.135085ms I0327 15:07:48.497411 3944 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-78c64d759d-4n9kt] creating logical port openshift-monitoring_metrics-server-78c64d759d-4n9kt for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:48.497542 3944 kube.go:256] Updating pod openshift-monitoring/metrics-server-78c64d759d-4n9kt I0327 15:07:48.515759 3944 pod.go:62] [openshift-monitoring/metrics-server-78c64d759d-4n9kt] pod update took 18.259278ms I0327 15:07:48.515782 3944 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-78c64d759d-4n9kt] addLogicalPort annotation time took 18.291349ms I0327 15:07:48.517348 3944 pods.go:271] [openshift-monitoring/metrics-server-78c64d759d-4n9kt] addLogicalPort took 19.959144ms, libovsdb time 1.142537ms W0327 15:07:48.715082 3944 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-659bf44899-rpc84 IP address from the namespace address-set, err: pod openshift-console/console-659bf44899-rpc84: no pod IPs found I0327 15:07:50.079817 3944 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:07:50.079949 3944 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0327 15:07:50.112407 3944 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 32.507964ms I0327 15:07:50.112429 3944 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 32.541782ms I0327 15:07:50.113731 3944 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 33.938493ms, libovsdb time 911.373µs I0327 15:07:51.128303 3944 cni.go:403] [openshift-monitoring/metrics-server-78c64d759d-4n9kt 73f973cccb9045830b96ff2bf38efa274542a45127548be639c80313c9b92664 network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-78c64d759d-4n9kt 73f973cccb9045830b96ff2bf38efa274542a45127548be639c80313c9b92664 network default NAD default] I0327 15:07:51.128303 3944 cni.go:403] [openshift-monitoring/prometheus-k8s-0 dc9741c1ff4b56b6fe98a0f39cf1ac060059d43b05b959fd0dbf7f9729623204 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 dc9741c1ff4b56b6fe98a0f39cf1ac060059d43b05b959fd0dbf7f9729623204 network default NAD default] I0327 15:07:51.128889 3944 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jjdwr 3a8be90f7e28c051cafb41690975dbca4d3db6536355cc08cb281cda8a0de551 network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jjdwr 3a8be90f7e28c051cafb41690975dbca4d3db6536355cc08cb281cda8a0de551 network default NAD default] I0327 15:07:51.140376 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: dc9741c1ff4b56b, network: default, NAD default, SandboxID: "dc9741c1ff4b56b6fe98a0f39cf1ac060059d43b05b959fd0dbf7f9729623204", PCI device ID: , UID: "cd25fe3b-b644-4712-9aa9-ac9d49418fa1", MAC: 0a:58:0a:82:00:14, IPs: [10.130.0.20/23] I0327 15:07:51.140900 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-78c64d759d-4n9kt, hostIfaceName: 73f973cccb90458, network: default, NAD default, SandboxID: "73f973cccb9045830b96ff2bf38efa274542a45127548be639c80313c9b92664", PCI device ID: , UID: "48dfb9b4-83ae-43c7-951b-3db8a9e27893", MAC: 0a:58:0a:82:00:13, IPs: [10.130.0.19/23] I0327 15:07:51.141424 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-jjdwr, hostIfaceName: 3a8be90f7e28c05, network: default, NAD default, SandboxID: "3a8be90f7e28c051cafb41690975dbca4d3db6536355cc08cb281cda8a0de551", PCI device ID: , UID: "9e7a0e20-2492-4ac0-977d-33a81bd33fa1", MAC: 0a:58:0a:82:00:12, IPs: [10.130.0.18/23] I0327 15:07:51.186744 3944 cni.go:424] [openshift-monitoring/prometheus-k8s-0 dc9741c1ff4b56b6fe98a0f39cf1ac060059d43b05b959fd0dbf7f9729623204 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 dc9741c1ff4b56b6fe98a0f39cf1ac060059d43b05b959fd0dbf7f9729623204 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:6b:0f:8e:67:0d\",\"name\":\"dc9741c1ff4b56b\"},{\"mac\":\"0a:58:0a:82:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/86cfab59-f348-4604-a86b-88e19d052caf\"}],\"ips\":[{\"address\":\"10.130.0.20/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:07:51.202899 3944 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jjdwr 3a8be90f7e28c051cafb41690975dbca4d3db6536355cc08cb281cda8a0de551 network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-jjdwr 3a8be90f7e28c051cafb41690975dbca4d3db6536355cc08cb281cda8a0de551 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:e2:53:5c:6c:31\",\"name\":\"3a8be90f7e28c05\"},{\"mac\":\"0a:58:0a:82:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/11255da8-80fa-40cd-b29a-9299eee733cf\"}],\"ips\":[{\"address\":\"10.130.0.18/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:07:51.220589 3944 cni.go:424] [openshift-monitoring/metrics-server-78c64d759d-4n9kt 73f973cccb9045830b96ff2bf38efa274542a45127548be639c80313c9b92664 network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-78c64d759d-4n9kt 73f973cccb9045830b96ff2bf38efa274542a45127548be639c80313c9b92664 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:16:66:f7:cd:90\",\"name\":\"73f973cccb90458\"},{\"mac\":\"0a:58:0a:82:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e9ef36b0-2acc-4e8a-961d-5d1a2cbd5ba3\"}],\"ips\":[{\"address\":\"10.130.0.19/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:07:51.262534 3944 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:07:51.262553 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:07:51.752403 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0327 15:07:51.779270 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0327 15:07:52.276087 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:07:52.308588 3944 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:52.308617 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:52.308626 3944 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:52.308635 3944 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:53.794578 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:07:53.837158 3944 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:07:53.837178 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:07:53.906380 3944 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:53.906427 3944 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:53.906435 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0327 15:07:55.882085 3944 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:55.882103 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:55.882112 3944 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0327 15:07:55.882117 3944 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0327 15:07:57.541137 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5877c497c6-7vmg7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:57.541757 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5877c497c6-7vmg7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:57.583258 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-65c4b7d4b8-946vl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:57.583805 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-65c4b7d4b8-946vl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:07:57.886024 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-65c4b7d4b8-946vl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:57.886075 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-65c4b7d4b8-946vl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:57.891432 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-65c4b7d4b8-946vl I0327 15:07:57.891458 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-65c4b7d4b8-946vl I0327 15:07:58.260534 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5877c497c6-7vmg7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:58.260565 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5877c497c6-7vmg7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:07:58.263999 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5877c497c6-7vmg7 I0327 15:07:58.264062 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5877c497c6-7vmg7 I0327 15:08:02.743389 3944 cni.go:403] [openshift-dns/dns-default-fnjs8 164e60394856c3a450a6fa7f89b76265837cfb54a792bb45cd81071e95796726 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-fnjs8 164e60394856c3a450a6fa7f89b76265837cfb54a792bb45cd81071e95796726 network default NAD default] I0327 15:08:02.751874 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-fnjs8, hostIfaceName: 164e60394856c3a, network: default, NAD default, SandboxID: "164e60394856c3a450a6fa7f89b76265837cfb54a792bb45cd81071e95796726", PCI device ID: , UID: "7945b36c-17ff-4a3a-a619-7463df1a09cc", MAC: 0a:58:0a:82:00:07, IPs: [10.130.0.7/23] I0327 15:08:02.795294 3944 cni.go:424] [openshift-dns/dns-default-fnjs8 164e60394856c3a450a6fa7f89b76265837cfb54a792bb45cd81071e95796726 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-fnjs8 164e60394856c3a450a6fa7f89b76265837cfb54a792bb45cd81071e95796726 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:ac:02:8f:bd:80\",\"name\":\"164e60394856c3a\"},{\"mac\":\"0a:58:0a:82:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f18b0101-b6ad-4cf1-bee3-94a56404722e\"}],\"ips\":[{\"address\":\"10.130.0.7/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:08:07.864470 3944 cni.go:403] [openshift-console/console-8579887566-h47gd 2f10f8f96a32f73f84fa9f9fc546b4c47996456e69ee9ea235b950d389bd38da network default NAD default] DEL starting CNI request [openshift-console/console-8579887566-h47gd 2f10f8f96a32f73f84fa9f9fc546b4c47996456e69ee9ea235b950d389bd38da network default NAD default] I0327 15:08:07.939378 3944 cni.go:424] [openshift-console/console-8579887566-h47gd 2f10f8f96a32f73f84fa9f9fc546b4c47996456e69ee9ea235b950d389bd38da network default NAD default] DEL finished CNI request [openshift-console/console-8579887566-h47gd 2f10f8f96a32f73f84fa9f9fc546b4c47996456e69ee9ea235b950d389bd38da network default NAD default], result "{}", err I0327 15:08:07.991142 3944 obj_retry.go:464] Detected object openshift-console/console-8579887566-h47gd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:07.991206 3944 pods.go:176] Deleting pod: openshift-console/console-8579887566-h47gd I0327 15:08:07.992442 3944 pods.go:236] Attempting to release IPs for pod: openshift-console/console-8579887566-h47gd, ips: 10.130.0.16 I0327 15:08:07.992471 3944 obj_retry.go:464] Detected object openshift-console/console-8579887566-h47gd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:08.003440 3944 base_network_controller_pods.go:487] [default/openshift-console/console-6865b86f9c-hl52k] creating logical port openshift-console_console-6865b86f9c-hl52k for pod on switch ip-10-0-0-80.ec2.internal I0327 15:08:08.003630 3944 kube.go:256] Updating pod openshift-console/console-6865b86f9c-hl52k I0327 15:08:08.017350 3944 pod.go:62] [openshift-console/console-6865b86f9c-hl52k] pod update took 13.747676ms I0327 15:08:08.017377 3944 base_network_controller_pods.go:937] [default/openshift-console/console-6865b86f9c-hl52k] addLogicalPort annotation time took 13.77868ms I0327 15:08:08.018677 3944 pods.go:271] [openshift-console/console-6865b86f9c-hl52k] addLogicalPort took 15.243339ms, libovsdb time 868.24µs I0327 15:08:08.396908 3944 cni.go:403] [openshift-console/console-6865b86f9c-hl52k 8d793fad0b9b052b9387d50146dc695d7dcc0f6c01a62a6636988d48ef5ade78 network default NAD default] ADD starting CNI request [openshift-console/console-6865b86f9c-hl52k 8d793fad0b9b052b9387d50146dc695d7dcc0f6c01a62a6636988d48ef5ade78 network default NAD default] I0327 15:08:08.405178 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6865b86f9c-hl52k, hostIfaceName: 8d793fad0b9b052, network: default, NAD default, SandboxID: "8d793fad0b9b052b9387d50146dc695d7dcc0f6c01a62a6636988d48ef5ade78", PCI device ID: , UID: "499f870d-857c-4c17-96af-bf6f720eb76b", MAC: 0a:58:0a:82:00:15, IPs: [10.130.0.21/23] I0327 15:08:08.452040 3944 cni.go:424] [openshift-console/console-6865b86f9c-hl52k 8d793fad0b9b052b9387d50146dc695d7dcc0f6c01a62a6636988d48ef5ade78 network default NAD default] ADD finished CNI request [openshift-console/console-6865b86f9c-hl52k 8d793fad0b9b052b9387d50146dc695d7dcc0f6c01a62a6636988d48ef5ade78 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:7e:0a:50:a9:b4\",\"name\":\"8d793fad0b9b052\"},{\"mac\":\"0a:58:0a:82:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c21c2394-7907-428c-9fe3-8cf6c2d2bced\"}],\"ips\":[{\"address\":\"10.130.0.21/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:08:08.856220 3944 obj_retry.go:458] Detected object openshift-console/console-8579887566-h47gd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:08.856243 3944 obj_retry.go:458] Detected object openshift-console/console-8579887566-h47gd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:08.861404 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8579887566-h47gd I0327 15:08:08.861433 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8579887566-h47gd W0327 15:08:13.239791 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:08:13.239819 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:08:14.784684 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0327 15:08:16.844549 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0327 15:08:16.844573 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0327 15:08:16.844583 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0327 15:08:17.133208 3944 obj_retry.go:464] Detected object openshift-console/console-659bf44899-57xvt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:17.133754 3944 obj_retry.go:464] Detected object openshift-console/console-659bf44899-57xvt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0327 15:08:17.987865 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0327 15:08:18.015892 3944 obj_retry.go:458] Detected object openshift-console/console-659bf44899-57xvt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:18.015924 3944 obj_retry.go:458] Detected object openshift-console/console-659bf44899-57xvt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:18.021541 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-659bf44899-57xvt I0327 15:08:18.021569 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-659bf44899-57xvt W0327 15:08:18.893493 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0327 15:08:27.917330 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 15:08:27.917377 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 15:08:27.917430 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0327 15:08:28.027086 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0327 15:08:28.929226 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0327 15:08:44.233465 3944 obj_retry.go:464] Detected object openshift-console/console-8579887566-55wtk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:44.234107 3944 obj_retry.go:464] Detected object openshift-console/console-8579887566-55wtk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:44.440987 3944 obj_retry.go:458] Detected object openshift-console/console-8579887566-55wtk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:44.441013 3944 obj_retry.go:458] Detected object openshift-console/console-8579887566-55wtk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:44.446776 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8579887566-55wtk I0327 15:08:44.446803 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8579887566-55wtk I0327 15:08:46.143974 3944 cni.go:403] [openshift-image-registry/image-registry-65c4b7d4b8-78rvv f4bcf45e02047d70caf5ec59989ab9dabf0271f51967882955bcfb1f2defd0b1 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-65c4b7d4b8-78rvv f4bcf45e02047d70caf5ec59989ab9dabf0271f51967882955bcfb1f2defd0b1 network default NAD default] I0327 15:08:46.215110 3944 cni.go:424] [openshift-image-registry/image-registry-65c4b7d4b8-78rvv f4bcf45e02047d70caf5ec59989ab9dabf0271f51967882955bcfb1f2defd0b1 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-65c4b7d4b8-78rvv f4bcf45e02047d70caf5ec59989ab9dabf0271f51967882955bcfb1f2defd0b1 network default NAD default], result "{}", err I0327 15:08:46.283311 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-65c4b7d4b8-78rvv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:46.283351 3944 pods.go:176] Deleting pod: openshift-image-registry/image-registry-65c4b7d4b8-78rvv I0327 15:08:46.284679 3944 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-65c4b7d4b8-78rvv, ips: 10.130.0.12 I0327 15:08:46.284708 3944 obj_retry.go:464] Detected object openshift-image-registry/image-registry-65c4b7d4b8-78rvv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:08:46.986025 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-65c4b7d4b8-78rvv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:46.986052 3944 obj_retry.go:458] Detected object openshift-image-registry/image-registry-65c4b7d4b8-78rvv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:08:46.991360 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-65c4b7d4b8-78rvv I0327 15:08:46.991395 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-65c4b7d4b8-78rvv W0327 15:08:50.553738 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:08:50.553765 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:08:50.553780 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:08:50.553788 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:08:51.049710 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:08:51.049734 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 15:08:55.835703 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0327 15:09:05.669197 3944 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:05.669905 3944 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:06.548195 3944 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:06.548221 3944 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:06.554296 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0327 15:09:06.554322 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0327 15:09:09.981353 3944 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:09.982047 3944 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:10.577367 3944 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:10.577389 3944 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:10.583788 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0327 15:09:10.583816 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0327 15:09:27.732024 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0327 15:09:37.763883 3944 obj_retry.go:464] Detected object openshift-console/console-6865b86f9c-xttgq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:37.764425 3944 obj_retry.go:464] Detected object openshift-console/console-6865b86f9c-xttgq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:38.310758 3944 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:38.311373 3944 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:38.355515 3944 obj_retry.go:458] Detected object openshift-console/console-6865b86f9c-xttgq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.355542 3944 obj_retry.go:458] Detected object openshift-console/console-6865b86f9c-xttgq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.365315 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6865b86f9c-xttgq I0327 15:09:38.365343 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6865b86f9c-xttgq I0327 15:09:38.638041 3944 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.638074 3944 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:38.643627 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0327 15:09:38.643650 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0327 15:09:38.841825 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=5 I0327 15:09:48.847880 3944 cni.go:403] [openshift-console/console-6865b86f9c-hl52k 8d793fad0b9b052b9387d50146dc695d7dcc0f6c01a62a6636988d48ef5ade78 network default NAD default] DEL starting CNI request [openshift-console/console-6865b86f9c-hl52k 8d793fad0b9b052b9387d50146dc695d7dcc0f6c01a62a6636988d48ef5ade78 network default NAD default] I0327 15:09:48.925121 3944 cni.go:424] [openshift-console/console-6865b86f9c-hl52k 8d793fad0b9b052b9387d50146dc695d7dcc0f6c01a62a6636988d48ef5ade78 network default NAD default] DEL finished CNI request [openshift-console/console-6865b86f9c-hl52k 8d793fad0b9b052b9387d50146dc695d7dcc0f6c01a62a6636988d48ef5ade78 network default NAD default], result "{}", err I0327 15:09:48.968862 3944 obj_retry.go:464] Detected object openshift-console/console-6865b86f9c-hl52k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:48.968903 3944 pods.go:176] Deleting pod: openshift-console/console-6865b86f9c-hl52k I0327 15:09:48.970214 3944 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6865b86f9c-hl52k, ips: 10.130.0.21 I0327 15:09:48.970245 3944 obj_retry.go:464] Detected object openshift-console/console-6865b86f9c-hl52k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:09:49.214226 3944 obj_retry.go:458] Detected object openshift-console/console-6865b86f9c-hl52k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:49.214261 3944 obj_retry.go:458] Detected object openshift-console/console-6865b86f9c-hl52k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:09:49.217159 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6865b86f9c-hl52k I0327 15:09:49.217192 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6865b86f9c-hl52k I0327 15:10:00.239487 3944 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29577070-gvnnq for pod on switch ip-10-0-0-80.ec2.internal I0327 15:10:00.239722 3944 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq I0327 15:10:00.265341 3944 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq] pod update took 25.719476ms I0327 15:10:00.265365 3944 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq] addLogicalPort annotation time took 25.751835ms I0327 15:10:00.266548 3944 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq] addLogicalPort took 27.083442ms, libovsdb time 873.666µs I0327 15:10:00.607020 3944 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq ed4a20cff9dfd6373c90b8bee3eccbd93d42a7f8da02052990c0d4d6f684e80d network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq ed4a20cff9dfd6373c90b8bee3eccbd93d42a7f8da02052990c0d4d6f684e80d network default NAD default] I0327 15:10:00.615373 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29577070-gvnnq, hostIfaceName: ed4a20cff9dfd63, network: default, NAD default, SandboxID: "ed4a20cff9dfd6373c90b8bee3eccbd93d42a7f8da02052990c0d4d6f684e80d", PCI device ID: , UID: "666a397e-ceb6-4661-a34f-432fb3574d42", MAC: 0a:58:0a:82:00:16, IPs: [10.130.0.22/23] I0327 15:10:00.658544 3944 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq ed4a20cff9dfd6373c90b8bee3eccbd93d42a7f8da02052990c0d4d6f684e80d network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq ed4a20cff9dfd6373c90b8bee3eccbd93d42a7f8da02052990c0d4d6f684e80d network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:5e:f5:33:0e:fb\",\"name\":\"ed4a20cff9dfd63\"},{\"mac\":\"0a:58:0a:82:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1163d38b-3d3c-4fae-b79d-30259af99b32\"}],\"ips\":[{\"address\":\"10.130.0.22/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:10:00.805448 3944 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:00.806021 3944 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:00.819653 3944 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:00.819693 3944 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:00.822842 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd I0327 15:10:00.822864 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577060-79dxd I0327 15:10:12.842086 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0327 15:10:13.035255 3944 cni.go:403] [openshift-monitoring/prometheus-k8s-0 dc9741c1ff4b56b6fe98a0f39cf1ac060059d43b05b959fd0dbf7f9729623204 network default NAD default] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 dc9741c1ff4b56b6fe98a0f39cf1ac060059d43b05b959fd0dbf7f9729623204 network default NAD default] I0327 15:10:13.105530 3944 cni.go:424] [openshift-monitoring/prometheus-k8s-0 dc9741c1ff4b56b6fe98a0f39cf1ac060059d43b05b959fd0dbf7f9729623204 network default NAD default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 dc9741c1ff4b56b6fe98a0f39cf1ac060059d43b05b959fd0dbf7f9729623204 network default NAD default], result "{}", err I0327 15:10:13.163204 3944 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:13.163239 3944 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0327 15:10:13.164603 3944 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.130.0.20 I0327 15:10:13.164632 3944 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:10:13.618211 3944 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:13.618238 3944 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:10:13.625945 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0327 15:10:13.625972 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0327 15:10:13.674821 3944 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:10:13.674985 3944 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0327 15:10:13.700031 3944 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 25.092036ms I0327 15:10:13.700068 3944 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 25.13659ms I0327 15:10:13.701392 3944 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 26.586575ms, libovsdb time 930.678µs I0327 15:10:14.035118 3944 cni.go:403] [openshift-monitoring/prometheus-k8s-0 096fcf24042a484beeece2bab040582658a1e8905a7607529793a72d62253256 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 096fcf24042a484beeece2bab040582658a1e8905a7607529793a72d62253256 network default NAD default] I0327 15:10:14.043383 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 096fcf24042a484, network: default, NAD default, SandboxID: "096fcf24042a484beeece2bab040582658a1e8905a7607529793a72d62253256", PCI device ID: , UID: "e9bba828-8a98-4177-a555-f49bce08cdda", MAC: 0a:58:0a:82:00:17, IPs: [10.130.0.23/23] I0327 15:10:14.086657 3944 cni.go:424] [openshift-monitoring/prometheus-k8s-0 096fcf24042a484beeece2bab040582658a1e8905a7607529793a72d62253256 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 096fcf24042a484beeece2bab040582658a1e8905a7607529793a72d62253256 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:9f:2a:6c:6f:b9\",\"name\":\"096fcf24042a484\"},{\"mac\":\"0a:58:0a:82:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/004aaa00-0d01-4a7c-ae98-9d6c61f370d9\"}],\"ips\":[{\"address\":\"10.130.0.23/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:10:14.824859 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0327 15:10:32.731859 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0327 15:10:38.726102 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=121 I0327 15:10:41.725248 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=57 I0327 15:10:47.853487 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0327 15:10:55.607895 3944 egressfirewall.go:628] Syncing node "ip-10-0-0-80.ec2.internal" for egress firewall I0327 15:10:55.607930 3944 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-80.ec2.internal I0327 15:10:55.607956 3944 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-80.ec2.internal: 30.67µs I0327 15:10:55.633158 3944 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-7j98b] creating logical port kube-system_global-pull-secret-syncer-7j98b for pod on switch ip-10-0-0-80.ec2.internal I0327 15:10:55.633287 3944 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-7j98b I0327 15:10:55.644394 3944 pod.go:62] [kube-system/global-pull-secret-syncer-7j98b] pod update took 11.139745ms I0327 15:10:55.644415 3944 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-7j98b] addLogicalPort annotation time took 11.165127ms I0327 15:10:55.645896 3944 pods.go:271] [kube-system/global-pull-secret-syncer-7j98b] addLogicalPort took 12.754833ms, libovsdb time 1.034489ms I0327 15:10:55.990704 3944 cni.go:403] [kube-system/global-pull-secret-syncer-7j98b f70205518d9ae629ea0ae0f17245cc1ba36a56651ac54e3f46b4efd29a8af8a6 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-7j98b f70205518d9ae629ea0ae0f17245cc1ba36a56651ac54e3f46b4efd29a8af8a6 network default NAD default] I0327 15:10:55.999383 3944 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-7j98b, hostIfaceName: f70205518d9ae62, network: default, NAD default, SandboxID: "f70205518d9ae629ea0ae0f17245cc1ba36a56651ac54e3f46b4efd29a8af8a6", PCI device ID: , UID: "ee14fb00-1ab5-45fb-aed4-d29cd72b2e64", MAC: 0a:58:0a:82:00:18, IPs: [10.130.0.24/23] I0327 15:10:56.042021 3944 cni.go:424] [kube-system/global-pull-secret-syncer-7j98b f70205518d9ae629ea0ae0f17245cc1ba36a56651ac54e3f46b4efd29a8af8a6 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-7j98b f70205518d9ae629ea0ae0f17245cc1ba36a56651ac54e3f46b4efd29a8af8a6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:1e:7e:0b:0e:c9\",\"name\":\"f70205518d9ae62\"},{\"mac\":\"0a:58:0a:82:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d8f83de4-a18e-4790-a0f5-e0473abe9339\"}],\"ips\":[{\"address\":\"10.130.0.24/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:10:58.691877 3944 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0327 15:11:09.730977 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0327 15:11:09.731041 3944 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4cnll IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-4cnll: no pod IPs found I0327 15:11:10.077199 3944 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0327 15:11:14.584412 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0327 15:11:14.584433 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0327 15:11:21.893546 3944 base_network_controller_pods.go:487] [default/openshift-console/console-5bf9956b89-qbvlb] creating logical port openshift-console_console-5bf9956b89-qbvlb for pod on switch ip-10-0-0-80.ec2.internal I0327 15:11:21.893645 3944 kube.go:256] Updating pod openshift-console/console-5bf9956b89-qbvlb I0327 15:11:21.937316 3944 pod.go:62] [openshift-console/console-5bf9956b89-qbvlb] pod update took 43.691119ms I0327 15:11:21.937347 3944 base_network_controller_pods.go:937] [default/openshift-console/console-5bf9956b89-qbvlb] addLogicalPort annotation time took 43.731408ms I0327 15:11:21.938636 3944 pods.go:271] [openshift-console/console-5bf9956b89-qbvlb] addLogicalPort took 45.102824ms, libovsdb time 881.966µs I0327 15:11:22.247658 3944 cni.go:403] [openshift-console/console-5bf9956b89-qbvlb b16767ce722792e60f5aeb3aa4d17176db69cab1a68afc8cfb382788716f3ee3 network default NAD default] ADD starting CNI request [openshift-console/console-5bf9956b89-qbvlb b16767ce722792e60f5aeb3aa4d17176db69cab1a68afc8cfb382788716f3ee3 network default NAD default] I0327 15:11:22.257983 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5bf9956b89-qbvlb, hostIfaceName: b16767ce722792e, network: default, NAD default, SandboxID: "b16767ce722792e60f5aeb3aa4d17176db69cab1a68afc8cfb382788716f3ee3", PCI device ID: , UID: "37323c53-8e94-4a15-b81f-18e9cb975410", MAC: 0a:58:0a:82:00:19, IPs: [10.130.0.25/23] I0327 15:11:22.300862 3944 cni.go:424] [openshift-console/console-5bf9956b89-qbvlb b16767ce722792e60f5aeb3aa4d17176db69cab1a68afc8cfb382788716f3ee3 network default NAD default] ADD finished CNI request [openshift-console/console-5bf9956b89-qbvlb b16767ce722792e60f5aeb3aa4d17176db69cab1a68afc8cfb382788716f3ee3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:5d:10:1d:b7:cf\",\"name\":\"b16767ce722792e\"},{\"mac\":\"0a:58:0a:82:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/13abf477-3a48-4a86-84cb-b783f0865293\"}],\"ips\":[{\"address\":\"10.130.0.25/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:11:32.620933 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0327 15:11:32.814370 3944 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0327 15:11:47.174736 3944 obj_retry.go:464] Detected object openshift-console/console-6b65cb69c9-6xjhr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:47.175360 3944 obj_retry.go:464] Detected object openshift-console/console-6b65cb69c9-6xjhr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:47.497869 3944 obj_retry.go:458] Detected object openshift-console/console-6b65cb69c9-6xjhr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:47.497913 3944 obj_retry.go:458] Detected object openshift-console/console-6b65cb69c9-6xjhr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:47.507739 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6b65cb69c9-6xjhr I0327 15:11:47.507778 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6b65cb69c9-6xjhr I0327 15:11:48.805167 3944 obj_retry.go:300] Retry object setup: *v1.Pod openshift-console/console-5bf9956b89-rslvj I0327 15:11:48.805199 3944 obj_retry.go:364] Adding new object: *v1.Pod openshift-console/console-5bf9956b89-rslvj I0327 15:11:48.805252 3944 obj_retry.go:375] Retry add failed for *v1.Pod openshift-console/console-5bf9956b89-rslvj, will try again later: failed to obtain IPs to add remote pod openshift-console/console-5bf9956b89-rslvj: suppressed error logged: pod openshift-console/console-5bf9956b89-rslvj: no pod IPs found I0327 15:11:57.917565 3944 obj_retry.go:464] Detected object openshift-console/console-6b65cb69c9-pgrnb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:57.918199 3944 obj_retry.go:464] Detected object openshift-console/console-6b65cb69c9-pgrnb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:11:58.268089 3944 obj_retry.go:458] Detected object openshift-console/console-6b65cb69c9-pgrnb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:58.268118 3944 obj_retry.go:458] Detected object openshift-console/console-6b65cb69c9-pgrnb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:11:58.277554 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6b65cb69c9-pgrnb I0327 15:11:58.277581 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6b65cb69c9-pgrnb I0327 15:11:58.819990 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0327 15:12:02.854371 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0327 15:12:11.740167 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=587 I0327 15:12:51.731624 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0327 15:13:31.813944 3944 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=11 I0327 15:13:36.848116 3944 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=11 I0327 15:15:16.838329 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0327 15:16:14.733799 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=7 I0327 15:16:36.847835 3944 namespace.go:98] [tsf] adding namespace I0327 15:16:36.848817 3944 namespace.go:102] [tsf] adding namespace took 957.327µs I0327 15:16:36.864887 3944 namespace.go:142] [tsf] updating namespace I0327 15:16:36.881065 3944 namespace.go:142] [tsf] updating namespace I0327 15:16:40.387537 3944 namespace.go:98] [cert-manager-operator] adding namespace I0327 15:16:40.388413 3944 namespace.go:102] [cert-manager-operator] adding namespace took 849.386µs I0327 15:16:40.391319 3944 namespace.go:98] [rhbk-operator] adding namespace I0327 15:16:40.391962 3944 namespace.go:102] [rhbk-operator] adding namespace took 624.112µs I0327 15:16:40.395075 3944 namespace.go:98] [konflux-ui] adding namespace I0327 15:16:40.395143 3944 namespace.go:98] [rhtpa-operator] adding namespace I0327 15:16:40.395284 3944 namespace.go:98] [tsf-tpa] adding namespace I0327 15:16:40.395428 3944 namespace.go:98] [konflux-operator] adding namespace I0327 15:16:40.395445 3944 namespace.go:98] [tsf-keycloak] adding namespace I0327 15:16:40.395505 3944 namespace.go:98] [tsf-tas] adding namespace I0327 15:16:40.395880 3944 namespace.go:102] [konflux-ui] adding namespace took 789.17µs I0327 15:16:40.396525 3944 namespace.go:102] [rhtpa-operator] adding namespace took 1.363622ms I0327 15:16:40.397129 3944 namespace.go:102] [tsf-tpa] adding namespace took 1.830773ms I0327 15:16:40.397648 3944 namespace.go:102] [konflux-operator] adding namespace took 2.20535ms I0327 15:16:40.398231 3944 namespace.go:102] [tsf-keycloak] adding namespace took 2.772907ms I0327 15:16:40.398868 3944 namespace.go:102] [tsf-tas] adding namespace took 3.339088ms I0327 15:16:40.404530 3944 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:40.422988 3944 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:40.426008 3944 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:40.442457 3944 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:40.448245 3944 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:40.451078 3944 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:40.451155 3944 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:40.457219 3944 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:40.461947 3944 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:40.461975 3944 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:40.463965 3944 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:16:40.465284 3944 namespace.go:142] [rhbk-operator] updating namespace I0327 15:16:40.479881 3944 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:40.494211 3944 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:16:40.501275 3944 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:40.514925 3944 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:40.522331 3944 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:40.541444 3944 namespace.go:142] [konflux-operator] updating namespace I0327 15:16:40.544762 3944 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:40.558597 3944 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:16:40.562781 3944 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:40.574537 3944 namespace.go:142] [tsf-tas] updating namespace I0327 15:16:40.580094 3944 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:40.589098 3944 namespace.go:142] [konflux-ui] updating namespace I0327 15:16:41.532716 3944 namespace.go:142] [tsf-tpa] updating namespace I0327 15:16:41.545183 3944 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:04.457050 3944 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:04.457649 3944 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:04.457696 3944 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:04.457705 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:04.457717 3944 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:04.457723 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:05.481600 3944 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:05.481630 3944 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:05.481648 3944 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:05.481659 3944 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf8vvbg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:06.481758 3944 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:06.482334 3944 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:06.482360 3944 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:06.482368 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:06.482377 3944 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:06.482381 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:07.503778 3944 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.504311 3944 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.504337 3944 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.504345 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:07.504354 3944 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.504358 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:07.515341 3944 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:07.515366 3944 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:07.515380 3944 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:07.515391 3944 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5gsbbb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:07.525375 3944 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.525883 3944 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.525908 3944 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.525916 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:07.525925 3944 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:07.525929 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:08.526936 3944 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.526957 3944 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.526973 3944 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.526982 3944 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1v4zz7 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.559230 3944 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.559249 3944 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.559265 3944 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:08.559274 3944 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8v4jq9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:09.518007 3944 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.518581 3944 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.518620 3944 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.518633 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:09.518646 3944 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.518652 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:09.549259 3944 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.552198 3944 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.552229 3944 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.552238 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0327 15:17:09.552248 3944 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0327 15:17:09.552252 3944 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0327 15:17:10.564686 3944 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.564714 3944 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.564735 3944 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.564745 3944 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf4qwqm of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.599818 3944 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.599836 3944 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.599856 3944 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:10.599868 3944 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7g27hf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:18.827174 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 15:17:20.945119 3944 namespace.go:98] [cert-manager] adding namespace I0327 15:17:20.945972 3944 namespace.go:102] [cert-manager] adding namespace took 831.664µs I0327 15:17:21.142074 3944 namespace.go:142] [cert-manager] updating namespace I0327 15:17:21.174683 3944 namespace.go:142] [cert-manager] updating namespace W0327 15:17:22.619122 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0327 15:17:22.777333 3944 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-7w68r] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-7w68r for pod on switch ip-10-0-0-80.ec2.internal I0327 15:17:22.777423 3944 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-7w68r I0327 15:17:22.806558 3944 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-7w68r] pod update took 29.158113ms I0327 15:17:22.806581 3944 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-7w68r] addLogicalPort annotation time took 29.186937ms I0327 15:17:22.807825 3944 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-7w68r] addLogicalPort took 30.50863ms, libovsdb time 875.697µs I0327 15:17:23.134168 3944 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-7w68r c5522cf3ba0c4649e87d3cb2f3cdc72abdfd5b96e43581731d0b834e5e078330 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-7w68r c5522cf3ba0c4649e87d3cb2f3cdc72abdfd5b96e43581731d0b834e5e078330 network default NAD default] I0327 15:17:23.143119 3944 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-7w68r, hostIfaceName: c5522cf3ba0c464, network: default, NAD default, SandboxID: "c5522cf3ba0c4649e87d3cb2f3cdc72abdfd5b96e43581731d0b834e5e078330", PCI device ID: , UID: "bc21c79b-0699-448b-9a66-7dbaf226728c", MAC: 0a:58:0a:82:00:1a, IPs: [10.130.0.26/23] I0327 15:17:23.188044 3944 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-7w68r c5522cf3ba0c4649e87d3cb2f3cdc72abdfd5b96e43581731d0b834e5e078330 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-7w68r c5522cf3ba0c4649e87d3cb2f3cdc72abdfd5b96e43581731d0b834e5e078330 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:37:75:85:8f:9b\",\"name\":\"c5522cf3ba0c464\"},{\"mac\":\"0a:58:0a:82:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/673c62f7-1d84-4823-9133-60196cadc9b6\"}],\"ips\":[{\"address\":\"10.130.0.26/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:17:24.569576 3944 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5b4bf68748-9fgvp] creating logical port cert-manager_cert-manager-cainjector-5b4bf68748-9fgvp for pod on switch ip-10-0-0-80.ec2.internal I0327 15:17:24.569708 3944 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5b4bf68748-9fgvp I0327 15:17:24.585909 3944 pod.go:62] [cert-manager/cert-manager-cainjector-5b4bf68748-9fgvp] pod update took 16.253435ms I0327 15:17:24.585936 3944 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-cainjector-5b4bf68748-9fgvp] addLogicalPort annotation time took 16.28724ms I0327 15:17:24.587429 3944 pods.go:271] [cert-manager/cert-manager-cainjector-5b4bf68748-9fgvp] addLogicalPort took 17.873158ms, libovsdb time 1.030733ms I0327 15:17:25.985339 3944 cni.go:403] [cert-manager/cert-manager-cainjector-5b4bf68748-9fgvp 1a1d4915b754904c9bd4d9d77b8898efc0dd864a6ab3f3b38a16f47965ab6ee4 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-9fgvp 1a1d4915b754904c9bd4d9d77b8898efc0dd864a6ab3f3b38a16f47965ab6ee4 network default NAD default] I0327 15:17:25.995207 3944 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5b4bf68748-9fgvp, hostIfaceName: 1a1d4915b754904, network: default, NAD default, SandboxID: "1a1d4915b754904c9bd4d9d77b8898efc0dd864a6ab3f3b38a16f47965ab6ee4", PCI device ID: , UID: "c369ac8b-456b-4ead-a057-a81a0002a514", MAC: 0a:58:0a:82:00:1b, IPs: [10.130.0.27/23] I0327 15:17:26.041013 3944 cni.go:424] [cert-manager/cert-manager-cainjector-5b4bf68748-9fgvp 1a1d4915b754904c9bd4d9d77b8898efc0dd864a6ab3f3b38a16f47965ab6ee4 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-9fgvp 1a1d4915b754904c9bd4d9d77b8898efc0dd864a6ab3f3b38a16f47965ab6ee4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:2f:1c:70:8b:db\",\"name\":\"1a1d4915b754904\"},{\"mac\":\"0a:58:0a:82:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2d128960-86aa-411c-8fc3-4afd3992c3dd\"}],\"ips\":[{\"address\":\"10.130.0.27/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:17:26.199905 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:17:26.212724 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:17:26.230042 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:17:26.256054 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0327 15:17:26.928072 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 15:17:26.928090 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 15:17:28.544440 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0327 15:17:29.399972 3944 namespace.go:142] [openshift-operators] updating namespace I0327 15:17:29.409754 3944 namespace.go:142] [openshift-operators] updating namespace I0327 15:17:32.733840 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=52 W0327 15:17:32.934169 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook W0327 15:17:32.934192 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook I0327 15:17:33.547199 3944 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59ccb4bb6b-k54qv] creating logical port cert-manager_cert-manager-59ccb4bb6b-k54qv for pod on switch ip-10-0-0-80.ec2.internal I0327 15:17:33.547295 3944 kube.go:256] Updating pod cert-manager/cert-manager-59ccb4bb6b-k54qv I0327 15:17:33.563420 3944 pod.go:62] [cert-manager/cert-manager-59ccb4bb6b-k54qv] pod update took 16.153267ms I0327 15:17:33.563449 3944 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-59ccb4bb6b-k54qv] addLogicalPort annotation time took 16.185988ms I0327 15:17:33.564913 3944 pods.go:271] [cert-manager/cert-manager-59ccb4bb6b-k54qv] addLogicalPort took 17.738097ms, libovsdb time 1.019856ms I0327 15:17:33.907444 3944 cni.go:403] [cert-manager/cert-manager-59ccb4bb6b-k54qv 783eb3f10e58cfa65d1841229806877dc0f7ba6244807125b07ec5713c640e55 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-59ccb4bb6b-k54qv 783eb3f10e58cfa65d1841229806877dc0f7ba6244807125b07ec5713c640e55 network default NAD default] I0327 15:17:33.916804 3944 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59ccb4bb6b-k54qv, hostIfaceName: 783eb3f10e58cfa, network: default, NAD default, SandboxID: "783eb3f10e58cfa65d1841229806877dc0f7ba6244807125b07ec5713c640e55", PCI device ID: , UID: "7c9e453e-be58-40f0-ad76-b90c1f63200c", MAC: 0a:58:0a:82:00:1c, IPs: [10.130.0.28/23] I0327 15:17:33.963237 3944 cni.go:424] [cert-manager/cert-manager-59ccb4bb6b-k54qv 783eb3f10e58cfa65d1841229806877dc0f7ba6244807125b07ec5713c640e55 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-59ccb4bb6b-k54qv 783eb3f10e58cfa65d1841229806877dc0f7ba6244807125b07ec5713c640e55 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:dc:9d:ce:fd:c1\",\"name\":\"783eb3f10e58cfa\"},{\"mac\":\"0a:58:0a:82:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d3669ace-5a15-467f-bdfb-567d2edb7db5\"}],\"ips\":[{\"address\":\"10.130.0.28/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:17:36.382141 3944 namespace.go:142] [openshift-operators] updating namespace I0327 15:17:36.948245 3944 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-m49rl] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-m49rl for pod on switch ip-10-0-0-80.ec2.internal I0327 15:17:36.948393 3944 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-m49rl I0327 15:17:36.983697 3944 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-m49rl] pod update took 35.304059ms I0327 15:17:36.983816 3944 base_network_controller_pods.go:937] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-m49rl] addLogicalPort annotation time took 35.462167ms I0327 15:17:36.985296 3944 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-m49rl] addLogicalPort took 37.075856ms, libovsdb time 974.498µs I0327 15:17:37.305459 3944 cni.go:403] [openshift-operators/tekton-operator-webhook-668b6fdd5d-m49rl d51ff6084b123734e8cd4ba04029e98cb05c525426d1bbf075748730434099ea network default NAD default] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-m49rl d51ff6084b123734e8cd4ba04029e98cb05c525426d1bbf075748730434099ea network default NAD default] I0327 15:17:37.314603 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-m49rl, hostIfaceName: d51ff6084b12373, network: default, NAD default, SandboxID: "d51ff6084b123734e8cd4ba04029e98cb05c525426d1bbf075748730434099ea", PCI device ID: , UID: "c849f58a-5004-48f6-bef7-91c3259720c5", MAC: 0a:58:0a:82:00:1d, IPs: [10.130.0.29/23] I0327 15:17:37.359886 3944 cni.go:424] [openshift-operators/tekton-operator-webhook-668b6fdd5d-m49rl d51ff6084b123734e8cd4ba04029e98cb05c525426d1bbf075748730434099ea network default NAD default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-m49rl d51ff6084b123734e8cd4ba04029e98cb05c525426d1bbf075748730434099ea network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:ec:0b:c2:1d:74\",\"name\":\"d51ff6084b12373\"},{\"mac\":\"0a:58:0a:82:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1c61f4b9-a444-4e9b-8717-081a35a1d4e9\"}],\"ips\":[{\"address\":\"10.130.0.29/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:17:45.911807 3944 namespace.go:98] [trusted-artifact-signer] adding namespace I0327 15:17:45.912685 3944 namespace.go:102] [trusted-artifact-signer] adding namespace took 831.895µs I0327 15:17:45.937229 3944 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:45.949550 3944 namespace.go:142] [trusted-artifact-signer] updating namespace W0327 15:17:46.662993 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0327 15:17:50.761501 3944 namespace.go:98] [openshift-pipelines] adding namespace I0327 15:17:50.762371 3944 namespace.go:102] [openshift-pipelines] adding namespace took 850.274µs I0327 15:17:50.776154 3944 namespace.go:142] [openshift-pipelines] updating namespace I0327 15:17:51.471104 3944 namespace.go:142] [cert-manager] updating namespace I0327 15:17:51.670858 3944 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:52.073712 3944 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:52.274225 3944 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:52.476006 3944 namespace.go:142] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] updating namespace I0327 15:17:52.878380 3944 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:53.097146 3944 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:53.308304 3944 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:53.516011 3944 namespace.go:142] [tsf] updating namespace I0327 15:17:53.710453 3944 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:53.912754 3944 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:54.117995 3944 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:54.242480 3944 namespace.go:142] [cert-manager] updating namespace I0327 15:17:54.252278 3944 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:54.259373 3944 namespace.go:142] [default] updating namespace I0327 15:17:54.269152 3944 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:54.276318 3944 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:54.286151 3944 namespace.go:142] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] updating namespace I0327 15:17:54.298225 3944 namespace.go:142] [openshift] updating namespace I0327 15:17:54.306084 3944 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:54.318297 3944 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:54.325319 3944 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:54.335259 3944 namespace.go:142] [tsf] updating namespace I0327 15:17:54.351315 3944 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:54.371592 3944 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:54.393564 3944 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:54.493140 3944 namespace.go:142] [cert-manager] updating namespace I0327 15:17:54.594571 3944 namespace.go:142] [cert-manager-operator] updating namespace I0327 15:17:54.694058 3944 namespace.go:142] [default] updating namespace I0327 15:17:54.794438 3944 namespace.go:142] [konflux-operator] updating namespace I0327 15:17:54.898610 3944 namespace.go:142] [konflux-ui] updating namespace I0327 15:17:54.996653 3944 namespace.go:142] [open-cluster-management-2pa4l1bb6ehlhnjl9imbt50cosjbq69n] updating namespace I0327 15:17:55.093460 3944 namespace.go:142] [openshift] updating namespace I0327 15:17:55.191568 3944 namespace.go:142] [rhbk-operator] updating namespace I0327 15:17:55.293932 3944 namespace.go:142] [rhtpa-operator] updating namespace I0327 15:17:55.393744 3944 namespace.go:142] [trusted-artifact-signer] updating namespace I0327 15:17:55.491398 3944 namespace.go:142] [tsf] updating namespace I0327 15:17:55.594193 3944 namespace.go:142] [tsf-keycloak] updating namespace I0327 15:17:55.699396 3944 namespace.go:142] [tsf-tas] updating namespace I0327 15:17:55.792944 3944 namespace.go:142] [tsf-tpa] updating namespace I0327 15:17:56.054628 3944 namespace.go:142] [openshift-pipelines] updating namespace I0327 15:17:56.744984 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=156 I0327 15:17:58.827030 3944 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:58.827603 3944 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:17:58.858166 3944 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:58.858189 3944 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:17:58.866885 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0327 15:17:58.866918 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0327 15:18:03.493290 3944 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4j8d8] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-4j8d8 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:18:03.493431 3944 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4j8d8 I0327 15:18:03.508232 3944 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4j8d8] pod update took 14.831328ms I0327 15:18:03.508258 3944 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4j8d8] addLogicalPort annotation time took 14.860865ms I0327 15:18:03.509648 3944 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4j8d8] addLogicalPort took 16.376781ms, libovsdb time 1.027889ms I0327 15:18:03.682402 3944 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-5ccds] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-5ccds for pod on switch ip-10-0-0-80.ec2.internal I0327 15:18:03.682524 3944 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-5ccds I0327 15:18:03.697319 3944 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-5ccds] pod update took 14.832791ms I0327 15:18:03.697346 3944 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-5ccds] addLogicalPort annotation time took 14.867044ms I0327 15:18:03.698623 3944 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-5ccds] addLogicalPort took 16.241314ms, libovsdb time 897.852µs I0327 15:18:03.864887 3944 cni.go:403] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4j8d8 ab04afd9f733339c33a7da594050c619a168cf700b9d86fa0f0dab72087484f2 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4j8d8 ab04afd9f733339c33a7da594050c619a168cf700b9d86fa0f0dab72087484f2 network default NAD default] I0327 15:18:03.873421 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-4j8d8, hostIfaceName: ab04afd9f733339, network: default, NAD default, SandboxID: "ab04afd9f733339c33a7da594050c619a168cf700b9d86fa0f0dab72087484f2", PCI device ID: , UID: "3fe76a38-29f9-4be5-91ba-dc2fb3edb6a3", MAC: 0a:58:0a:82:00:1e, IPs: [10.130.0.30/23] I0327 15:18:03.918783 3944 cni.go:424] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4j8d8 ab04afd9f733339c33a7da594050c619a168cf700b9d86fa0f0dab72087484f2 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-4j8d8 ab04afd9f733339c33a7da594050c619a168cf700b9d86fa0f0dab72087484f2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:6e:7d:4a:f0:1e\",\"name\":\"ab04afd9f733339\"},{\"mac\":\"0a:58:0a:82:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b5130f2c-b359-45cd-88b1-3fe574a40eff\"}],\"ips\":[{\"address\":\"10.130.0.30/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:18:04.042341 3944 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-5ccds 88ce2ba51fc1dfb446028fa0234ffd67dc5d80973d243f611973a8f25c6501a3 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-5ccds 88ce2ba51fc1dfb446028fa0234ffd67dc5d80973d243f611973a8f25c6501a3 network default NAD default] I0327 15:18:04.050877 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-5ccds, hostIfaceName: 88ce2ba51fc1dfb, network: default, NAD default, SandboxID: "88ce2ba51fc1dfb446028fa0234ffd67dc5d80973d243f611973a8f25c6501a3", PCI device ID: , UID: "2676234c-219b-4271-adff-e5a2cdfe66bf", MAC: 0a:58:0a:82:00:1f, IPs: [10.130.0.31/23] I0327 15:18:04.096485 3944 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-5ccds 88ce2ba51fc1dfb446028fa0234ffd67dc5d80973d243f611973a8f25c6501a3 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-5ccds 88ce2ba51fc1dfb446028fa0234ffd67dc5d80973d243f611973a8f25c6501a3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:c7:ee:c8:44:25\",\"name\":\"88ce2ba51fc1dfb\"},{\"mac\":\"0a:58:0a:82:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f32c931c-6b68-4aca-ac8c-260993f2a103\"}],\"ips\":[{\"address\":\"10.130.0.31/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:18:06.782124 3944 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0327 15:18:07.794978 3944 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:18:07.794995 3944 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:18:07.795000 3944 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0327 15:18:08.121639 3944 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:18:08.121660 3944 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:18:08.121678 3944 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:18:10.793194 3944 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:18:10.793213 3944 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:18:10.793219 3944 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:18:10.793224 3944 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0327 15:18:11.796111 3944 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0327 15:18:11.840141 3944 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0327 15:18:19.129165 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:18:19.129191 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0327 15:18:19.129199 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0327 15:18:26.843905 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0327 15:18:28.563643 3944 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-wwfgx] creating logical port openshift-pipelines_tekton-triggers-controller-68564d75ff-wwfgx for pod on switch ip-10-0-0-80.ec2.internal I0327 15:18:28.563746 3944 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-68564d75ff-wwfgx I0327 15:18:28.591365 3944 pod.go:62] [openshift-pipelines/tekton-triggers-controller-68564d75ff-wwfgx] pod update took 27.647791ms I0327 15:18:28.591390 3944 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-wwfgx] addLogicalPort annotation time took 27.674658ms I0327 15:18:28.592663 3944 pods.go:271] [openshift-pipelines/tekton-triggers-controller-68564d75ff-wwfgx] addLogicalPort took 29.035155ms, libovsdb time 890.028µs I0327 15:18:28.797820 3944 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-9jqnf] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-8bb674894-9jqnf for pod on switch ip-10-0-0-80.ec2.internal I0327 15:18:28.797910 3944 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-9jqnf I0327 15:18:28.812761 3944 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-9jqnf] pod update took 14.872426ms I0327 15:18:28.812782 3944 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-9jqnf] addLogicalPort annotation time took 14.899071ms I0327 15:18:28.814141 3944 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-9jqnf] addLogicalPort took 16.336682ms, libovsdb time 991.368µs I0327 15:18:28.921190 3944 cni.go:403] [openshift-pipelines/tekton-triggers-controller-68564d75ff-wwfgx 552bab441ed3e1e30cbca2f96b742ec0214e2ec578b1902d5e325dd2ec056b85 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-wwfgx 552bab441ed3e1e30cbca2f96b742ec0214e2ec578b1902d5e325dd2ec056b85 network default NAD default] I0327 15:18:28.929633 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-68564d75ff-wwfgx, hostIfaceName: 552bab441ed3e1e, network: default, NAD default, SandboxID: "552bab441ed3e1e30cbca2f96b742ec0214e2ec578b1902d5e325dd2ec056b85", PCI device ID: , UID: "f7f7e2cb-40c0-4246-84c7-c96b62214d62", MAC: 0a:58:0a:82:00:20, IPs: [10.130.0.32/23] I0327 15:18:28.972689 3944 cni.go:424] [openshift-pipelines/tekton-triggers-controller-68564d75ff-wwfgx 552bab441ed3e1e30cbca2f96b742ec0214e2ec578b1902d5e325dd2ec056b85 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-wwfgx 552bab441ed3e1e30cbca2f96b742ec0214e2ec578b1902d5e325dd2ec056b85 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d2:2b:bb:ce:7b:97\",\"name\":\"552bab441ed3e1e\"},{\"mac\":\"0a:58:0a:82:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eb3654ff-679a-4643-b098-999e7a0f51ef\"}],\"ips\":[{\"address\":\"10.130.0.32/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:18:29.156937 3944 cni.go:403] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-9jqnf 57bbd7a7855c946d1797c07888534b50266ae13862d9ef10f7524b0c9441b63f network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-9jqnf 57bbd7a7855c946d1797c07888534b50266ae13862d9ef10f7524b0c9441b63f network default NAD default] I0327 15:18:29.165352 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-8bb674894-9jqnf, hostIfaceName: 57bbd7a7855c946, network: default, NAD default, SandboxID: "57bbd7a7855c946d1797c07888534b50266ae13862d9ef10f7524b0c9441b63f", PCI device ID: , UID: "14522e19-9fe0-4f52-a7c0-33a82a61bd52", MAC: 0a:58:0a:82:00:21, IPs: [10.130.0.33/23] I0327 15:18:29.209125 3944 cni.go:424] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-9jqnf 57bbd7a7855c946d1797c07888534b50266ae13862d9ef10f7524b0c9441b63f network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-9jqnf 57bbd7a7855c946d1797c07888534b50266ae13862d9ef10f7524b0c9441b63f network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:42:64:bc:9e:7a\",\"name\":\"57bbd7a7855c946\"},{\"mac\":\"0a:58:0a:82:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bc5d2fe0-5219-48dc-849f-ba802d36cc76\"}],\"ips\":[{\"address\":\"10.130.0.33/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:18:32.224568 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 15:18:34.853481 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0327 15:18:36.028052 3944 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:36.028638 3944 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:18:36.049089 3944 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:18:36.049112 3944 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:18:36.055735 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0327 15:18:36.055755 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0327 15:18:41.728700 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=75 W0327 15:18:42.948312 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:18:42.948314 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:42.948377 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:18:43.231717 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0327 15:18:45.093368 3944 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-grfkz] creating logical port openshift-pipelines_tekton-chains-controller-5fb999ffff-grfkz for pod on switch ip-10-0-0-80.ec2.internal I0327 15:18:45.093493 3944 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-5fb999ffff-grfkz I0327 15:18:45.110149 3944 pod.go:62] [openshift-pipelines/tekton-chains-controller-5fb999ffff-grfkz] pod update took 16.69414ms I0327 15:18:45.110251 3944 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-grfkz] addLogicalPort annotation time took 16.791503ms I0327 15:18:45.111723 3944 pods.go:271] [openshift-pipelines/tekton-chains-controller-5fb999ffff-grfkz] addLogicalPort took 18.37632ms, libovsdb time 1.086417ms I0327 15:18:45.464039 3944 cni.go:403] [openshift-pipelines/tekton-chains-controller-5fb999ffff-grfkz 9c9aad9269bfc04b192986e1e10c298fffa21ffa68fa0bc661b8d1b6246945b3 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-grfkz 9c9aad9269bfc04b192986e1e10c298fffa21ffa68fa0bc661b8d1b6246945b3 network default NAD default] I0327 15:18:45.472506 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-5fb999ffff-grfkz, hostIfaceName: 9c9aad9269bfc04, network: default, NAD default, SandboxID: "9c9aad9269bfc04b192986e1e10c298fffa21ffa68fa0bc661b8d1b6246945b3", PCI device ID: , UID: "ba2dd457-4050-49e2-8bea-954e43f2e607", MAC: 0a:58:0a:82:00:22, IPs: [10.130.0.34/23] I0327 15:18:45.519009 3944 cni.go:424] [openshift-pipelines/tekton-chains-controller-5fb999ffff-grfkz 9c9aad9269bfc04b192986e1e10c298fffa21ffa68fa0bc661b8d1b6246945b3 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-grfkz 9c9aad9269bfc04b192986e1e10c298fffa21ffa68fa0bc661b8d1b6246945b3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:72:dd:8f:05:fd\",\"name\":\"9c9aad9269bfc04\"},{\"mac\":\"0a:58:0a:82:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1f8aaa6f-93b1-4de1-ac77-01da9cb101be\"}],\"ips\":[{\"address\":\"10.130.0.34/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:18:50.182515 3944 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0327 15:19:03.328734 3944 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:03.329333 3944 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:04.076238 3944 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:04.076262 3944 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:04.081810 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 15:19:04.081836 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0327 15:19:04.440826 3944 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-jqk9p] creating logical port openshift-pipelines_tekton-results-api-7b4b7bb78c-jqk9p for pod on switch ip-10-0-0-80.ec2.internal I0327 15:19:04.441043 3944 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-7b4b7bb78c-jqk9p I0327 15:19:04.455894 3944 pod.go:62] [openshift-pipelines/tekton-results-api-7b4b7bb78c-jqk9p] pod update took 14.894196ms I0327 15:19:04.455916 3944 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-jqk9p] addLogicalPort annotation time took 14.91985ms I0327 15:19:04.457282 3944 pods.go:271] [openshift-pipelines/tekton-results-api-7b4b7bb78c-jqk9p] addLogicalPort took 16.475135ms, libovsdb time 983.878µs I0327 15:19:04.799403 3944 cni.go:403] [openshift-pipelines/tekton-results-api-7b4b7bb78c-jqk9p 85e95f33875034a7b1c1f9d86d057206314cdc6cb44890a3361510ec65734dcd network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-jqk9p 85e95f33875034a7b1c1f9d86d057206314cdc6cb44890a3361510ec65734dcd network default NAD default] I0327 15:19:04.807994 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-7b4b7bb78c-jqk9p, hostIfaceName: 85e95f33875034a, network: default, NAD default, SandboxID: "85e95f33875034a7b1c1f9d86d057206314cdc6cb44890a3361510ec65734dcd", PCI device ID: , UID: "89794c18-20f3-499c-b15f-b863495461ee", MAC: 0a:58:0a:82:00:23, IPs: [10.130.0.35/23] I0327 15:19:04.855290 3944 cni.go:424] [openshift-pipelines/tekton-results-api-7b4b7bb78c-jqk9p 85e95f33875034a7b1c1f9d86d057206314cdc6cb44890a3361510ec65734dcd network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-jqk9p 85e95f33875034a7b1c1f9d86d057206314cdc6cb44890a3361510ec65734dcd network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:61:ab:dd:5a:9c\",\"name\":\"85e95f33875034a\"},{\"mac\":\"0a:58:0a:82:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/01085dd1-9f85-428b-a91b-2eb356214b20\"}],\"ips\":[{\"address\":\"10.130.0.35/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:19:06.107959 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:06.107981 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:06.108455 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:19:09.402155 3944 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:19:09.402177 3944 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:19:09.402184 3944 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:19:15.436838 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:19:15.436861 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0327 15:19:15.436867 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0327 15:19:21.502437 3944 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-mlwjq] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-mlwjq for pod on switch ip-10-0-0-80.ec2.internal I0327 15:19:21.502541 3944 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-mlwjq I0327 15:19:21.517101 3944 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-mlwjq] pod update took 14.58649ms I0327 15:19:21.517201 3944 base_network_controller_pods.go:937] [default/openshift-pipelines/tkn-cli-serve-784b566968-mlwjq] addLogicalPort annotation time took 14.691392ms I0327 15:19:21.520033 3944 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-mlwjq] addLogicalPort took 17.620433ms, libovsdb time 1.351501ms I0327 15:19:21.861770 3944 cni.go:403] [openshift-pipelines/tkn-cli-serve-784b566968-mlwjq 26bee46d580c3a8e9cac99f9ff95978456d50daf942415641bdeb7274e0c13dc network default NAD default] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-mlwjq 26bee46d580c3a8e9cac99f9ff95978456d50daf942415641bdeb7274e0c13dc network default NAD default] I0327 15:19:21.870627 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-mlwjq, hostIfaceName: 26bee46d580c3a8, network: default, NAD default, SandboxID: "26bee46d580c3a8e9cac99f9ff95978456d50daf942415641bdeb7274e0c13dc", PCI device ID: , UID: "8b910447-8d0f-4ca1-a843-39f7859bf0e3", MAC: 0a:58:0a:82:00:24, IPs: [10.130.0.36/23] I0327 15:19:21.917031 3944 cni.go:424] [openshift-pipelines/tkn-cli-serve-784b566968-mlwjq 26bee46d580c3a8e9cac99f9ff95978456d50daf942415641bdeb7274e0c13dc network default NAD default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-mlwjq 26bee46d580c3a8e9cac99f9ff95978456d50daf942415641bdeb7274e0c13dc network default NAD default], result "{\"interfaces\":[{\"mac\":\"4a:38:b6:43:67:90\",\"name\":\"26bee46d580c3a8\"},{\"mac\":\"0a:58:0a:82:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/48d7ce3a-27ad-4042-8f7a-48a2324f15ff\"}],\"ips\":[{\"address\":\"10.130.0.36/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:19:24.727870 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=61 I0327 15:19:25.856860 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0327 15:19:34.527301 3944 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-62rhl] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-62rhl for pod on switch ip-10-0-0-80.ec2.internal I0327 15:19:34.527401 3944 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-62rhl I0327 15:19:34.539193 3944 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-62rhl] pod update took 11.815019ms I0327 15:19:34.539217 3944 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-62rhl] addLogicalPort annotation time took 11.846388ms I0327 15:19:34.540487 3944 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-62rhl] addLogicalPort took 13.199614ms, libovsdb time 848.182µs I0327 15:19:34.888356 3944 cni.go:403] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-62rhl d59b1437e4e6da70ac813181769d4b3cb89609fafd5b34f7cbbb4cb3a8d99ba8 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-62rhl d59b1437e4e6da70ac813181769d4b3cb89609fafd5b34f7cbbb4cb3a8d99ba8 network default NAD default] I0327 15:19:34.896474 3944 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-62rhl, hostIfaceName: d59b1437e4e6da7, network: default, NAD default, SandboxID: "d59b1437e4e6da70ac813181769d4b3cb89609fafd5b34f7cbbb4cb3a8d99ba8", PCI device ID: , UID: "8e2f7a1f-2dba-456f-afcc-3ff2b0288e1f", MAC: 0a:58:0a:82:00:25, IPs: [10.130.0.37/23] I0327 15:19:35.148375 3944 cni.go:424] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-62rhl d59b1437e4e6da70ac813181769d4b3cb89609fafd5b34f7cbbb4cb3a8d99ba8 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-62rhl d59b1437e4e6da70ac813181769d4b3cb89609fafd5b34f7cbbb4cb3a8d99ba8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:26:b4:45:06:a4\",\"name\":\"d59b1437e4e6da7\"},{\"mac\":\"0a:58:0a:82:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2b518d8b-3a8b-46e9-bce2-c1e8732b1983\"}],\"ips\":[{\"address\":\"10.130.0.37/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:19:37.535461 3944 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0327 15:19:38.493720 3944 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:38.494325 3944 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:41.734093 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0327 15:19:44.489529 3944 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-bflzg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:44.490143 3944 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-bflzg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:45.513968 3944 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-bflzg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:45.513996 3944 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-bflzg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:45.869875 3944 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:45.870481 3944 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:19:46.360745 3944 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:46.360775 3944 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:19:46.366684 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0327 15:19:46.366707 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0327 15:19:48.384469 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0327 15:19:48.387169 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:48.387182 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0327 15:19:48.541952 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0327 15:19:55.849735 3944 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0327 15:20:00.647229 3944 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq ed4a20cff9dfd6373c90b8bee3eccbd93d42a7f8da02052990c0d4d6f684e80d network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq ed4a20cff9dfd6373c90b8bee3eccbd93d42a7f8da02052990c0d4d6f684e80d network default NAD default] I0327 15:20:00.711819 3944 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq ed4a20cff9dfd6373c90b8bee3eccbd93d42a7f8da02052990c0d4d6f684e80d network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq ed4a20cff9dfd6373c90b8bee3eccbd93d42a7f8da02052990c0d4d6f684e80d network default NAD default], result "{}", err I0327 15:20:01.646125 3944 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:01.646168 3944 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq I0327 15:20:01.647661 3944 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq, ips: 10.130.0.22 I0327 15:20:01.647706 3944 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:01.654512 3944 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:01.654567 3944 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:01.661398 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq I0327 15:20:01.661426 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29577070-gvnnq I0327 15:20:05.645636 3944 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-pjn4l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:05.646213 3944 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-pjn4l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:06.530727 3944 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-pjn4l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:06.530762 3944 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577080-pjn4l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:15.815750 3944 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0327 15:20:38.609647 3944 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:38.609688 3944 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:38.615030 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 15:20:38.615051 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 15:20:40.855884 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0327 15:20:42.817085 3944 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0327 15:20:46.878543 3944 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:46.879162 3944 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:20:46.900773 3944 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:46.900795 3944 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:20:46.906338 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0327 15:20:46.906362 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0327 15:20:49.472930 3944 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-4hn7z-bj6gp] creating logical port tsf-tas_ctlog-createtree-job-4hn7z-bj6gp for pod on switch ip-10-0-0-80.ec2.internal I0327 15:20:49.473016 3944 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-4hn7z-bj6gp I0327 15:20:49.484859 3944 pod.go:62] [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp] pod update took 11.862538ms I0327 15:20:49.484881 3944 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-createtree-job-4hn7z-bj6gp] addLogicalPort annotation time took 11.889058ms I0327 15:20:49.486105 3944 pods.go:271] [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp] addLogicalPort took 13.195991ms, libovsdb time 848.109µs I0327 15:20:49.679782 3944 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-6cdcf85b7f-vrqw6] creating logical port tsf-tas_trillian-logsigner-6cdcf85b7f-vrqw6 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:20:49.679873 3944 kube.go:256] Updating pod tsf-tas/trillian-logsigner-6cdcf85b7f-vrqw6 I0327 15:20:49.693432 3944 pod.go:62] [tsf-tas/trillian-logsigner-6cdcf85b7f-vrqw6] pod update took 13.583255ms I0327 15:20:49.693455 3944 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logsigner-6cdcf85b7f-vrqw6] addLogicalPort annotation time took 13.613821ms I0327 15:20:49.694846 3944 pods.go:271] [tsf-tas/trillian-logsigner-6cdcf85b7f-vrqw6] addLogicalPort took 15.079993ms, libovsdb time 890.373µs I0327 15:20:49.818414 3944 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-c4mtb-wb5d4] creating logical port tsf-tas_rekor-createtree-job-c4mtb-wb5d4 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:20:49.818530 3944 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-c4mtb-wb5d4 I0327 15:20:49.829434 3944 pod.go:62] [tsf-tas/rekor-createtree-job-c4mtb-wb5d4] pod update took 10.938099ms I0327 15:20:49.829473 3944 base_network_controller_pods.go:937] [default/tsf-tas/rekor-createtree-job-c4mtb-wb5d4] addLogicalPort annotation time took 10.983499ms I0327 15:20:49.831143 3944 pods.go:271] [tsf-tas/rekor-createtree-job-c4mtb-wb5d4] addLogicalPort took 12.748828ms, libovsdb time 1.104252ms I0327 15:20:49.833136 3944 cni.go:403] [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp 3172e65f88229863c5f885c577459922e7efe8c900a7d9246af791a981b33f75 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp 3172e65f88229863c5f885c577459922e7efe8c900a7d9246af791a981b33f75 network default NAD default] I0327 15:20:49.843579 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-4hn7z-bj6gp, hostIfaceName: 3172e65f8822986, network: default, NAD default, SandboxID: "3172e65f88229863c5f885c577459922e7efe8c900a7d9246af791a981b33f75", PCI device ID: , UID: "15492cd8-808a-4fa6-86ac-fd765c9f296b", MAC: 0a:58:0a:82:00:26, IPs: [10.130.0.38/23] I0327 15:20:49.891021 3944 cni.go:424] [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp 3172e65f88229863c5f885c577459922e7efe8c900a7d9246af791a981b33f75 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp 3172e65f88229863c5f885c577459922e7efe8c900a7d9246af791a981b33f75 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:e1:5e:ca:2f:85\",\"name\":\"3172e65f8822986\"},{\"mac\":\"0a:58:0a:82:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67cf2cb0-fc84-4f3e-b45d-e654ae589021\"}],\"ips\":[{\"address\":\"10.130.0.38/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:20:50.184015 3944 cni.go:403] [tsf-tas/rekor-createtree-job-c4mtb-wb5d4 f7aa7dc85bb7c953f1bb01a477af31b744048db79c2c850176acdc8cdd1c1eb3 network default NAD default] ADD starting CNI request [tsf-tas/rekor-createtree-job-c4mtb-wb5d4 f7aa7dc85bb7c953f1bb01a477af31b744048db79c2c850176acdc8cdd1c1eb3 network default NAD default] I0327 15:20:50.192792 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-c4mtb-wb5d4, hostIfaceName: f7aa7dc85bb7c95, network: default, NAD default, SandboxID: "f7aa7dc85bb7c953f1bb01a477af31b744048db79c2c850176acdc8cdd1c1eb3", PCI device ID: , UID: "8870589d-a440-4677-b303-78677bf952b5", MAC: 0a:58:0a:82:00:28, IPs: [10.130.0.40/23] I0327 15:20:50.238539 3944 cni.go:424] [tsf-tas/rekor-createtree-job-c4mtb-wb5d4 f7aa7dc85bb7c953f1bb01a477af31b744048db79c2c850176acdc8cdd1c1eb3 network default NAD default] ADD finished CNI request [tsf-tas/rekor-createtree-job-c4mtb-wb5d4 f7aa7dc85bb7c953f1bb01a477af31b744048db79c2c850176acdc8cdd1c1eb3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:58:e0:01:53:7f\",\"name\":\"f7aa7dc85bb7c95\"},{\"mac\":\"0a:58:0a:82:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d23068ce-4638-4874-9f92-8ddbce405218\"}],\"ips\":[{\"address\":\"10.130.0.40/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:20:50.648109 3944 cni.go:403] [tsf-tas/trillian-logsigner-6cdcf85b7f-vrqw6 e1c0f893da449ff8c0ffab1ee4ba51822eb07115bc9e640fcf51cc803c40548f network default NAD default] ADD starting CNI request [tsf-tas/trillian-logsigner-6cdcf85b7f-vrqw6 e1c0f893da449ff8c0ffab1ee4ba51822eb07115bc9e640fcf51cc803c40548f network default NAD default] I0327 15:20:50.658570 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-6cdcf85b7f-vrqw6, hostIfaceName: e1c0f893da449ff, network: default, NAD default, SandboxID: "e1c0f893da449ff8c0ffab1ee4ba51822eb07115bc9e640fcf51cc803c40548f", PCI device ID: , UID: "8525cbc0-8010-4c9f-9dc6-f3e1292de198", MAC: 0a:58:0a:82:00:27, IPs: [10.130.0.39/23] I0327 15:20:50.715832 3944 cni.go:424] [tsf-tas/trillian-logsigner-6cdcf85b7f-vrqw6 e1c0f893da449ff8c0ffab1ee4ba51822eb07115bc9e640fcf51cc803c40548f network default NAD default] ADD finished CNI request [tsf-tas/trillian-logsigner-6cdcf85b7f-vrqw6 e1c0f893da449ff8c0ffab1ee4ba51822eb07115bc9e640fcf51cc803c40548f network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:58:c3:b7:02:2c\",\"name\":\"e1c0f893da449ff\"},{\"mac\":\"0a:58:0a:82:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ed816077-c2c3-4366-9969-3653a9aacc69\"}],\"ips\":[{\"address\":\"10.130.0.39/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:20:56.649004 3944 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0327 15:20:56.918331 3944 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0327 15:20:58.866903 3944 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0327 15:20:58.866930 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0327 15:21:02.005320 3944 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ns4gd-lf892 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:02.005952 3944 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-ns4gd-lf892 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:02.883470 3944 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ns4gd-lf892 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:02.883495 3944 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-ns4gd-lf892 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0327 15:21:03.689018 3944 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0327 15:21:08.792728 3944 namespace.go:142] [openshift-pipelines] updating namespace W0327 15:21:30.051707 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0327 15:21:42.244063 3944 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:42.244622 3944 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:42.262045 3944 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:42.262092 3944 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:42.265331 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0327 15:21:42.265360 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0327 15:21:44.152528 3944 cni.go:403] [tsf-tas/rekor-createtree-job-c4mtb-wb5d4 f7aa7dc85bb7c953f1bb01a477af31b744048db79c2c850176acdc8cdd1c1eb3 network default NAD default] DEL starting CNI request [tsf-tas/rekor-createtree-job-c4mtb-wb5d4 f7aa7dc85bb7c953f1bb01a477af31b744048db79c2c850176acdc8cdd1c1eb3 network default NAD default] I0327 15:21:44.242043 3944 cni.go:424] [tsf-tas/rekor-createtree-job-c4mtb-wb5d4 f7aa7dc85bb7c953f1bb01a477af31b744048db79c2c850176acdc8cdd1c1eb3 network default NAD default] DEL finished CNI request [tsf-tas/rekor-createtree-job-c4mtb-wb5d4 f7aa7dc85bb7c953f1bb01a477af31b744048db79c2c850176acdc8cdd1c1eb3 network default NAD default], result "{}", err I0327 15:21:44.295352 3944 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-c4mtb-wb5d4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:44.295384 3944 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-c4mtb-wb5d4 I0327 15:21:44.296791 3944 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-c4mtb-wb5d4, ips: 10.130.0.40 I0327 15:21:44.296823 3944 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-c4mtb-wb5d4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:44.822340 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0327 15:21:45.158375 3944 cni.go:403] [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp 3172e65f88229863c5f885c577459922e7efe8c900a7d9246af791a981b33f75 network default NAD default] DEL starting CNI request [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp 3172e65f88229863c5f885c577459922e7efe8c900a7d9246af791a981b33f75 network default NAD default] I0327 15:21:45.163794 3944 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-c4mtb-wb5d4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:45.163866 3944 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-c4mtb-wb5d4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:45.240378 3944 cni.go:424] [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp 3172e65f88229863c5f885c577459922e7efe8c900a7d9246af791a981b33f75 network default NAD default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-4hn7z-bj6gp 3172e65f88229863c5f885c577459922e7efe8c900a7d9246af791a981b33f75 network default NAD default], result "{}", err I0327 15:21:45.286372 3944 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-4hn7z-bj6gp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:45.286412 3944 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-4hn7z-bj6gp I0327 15:21:45.287892 3944 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-4hn7z-bj6gp, ips: 10.130.0.38 I0327 15:21:45.287923 3944 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-4hn7z-bj6gp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:45.846523 3944 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-7bf4d4b9fb-ww7w4] creating logical port tsf-tas_rekor-redis-7bf4d4b9fb-ww7w4 for pod on switch ip-10-0-0-80.ec2.internal I0327 15:21:45.846661 3944 kube.go:256] Updating pod tsf-tas/rekor-redis-7bf4d4b9fb-ww7w4 I0327 15:21:45.859837 3944 pod.go:62] [tsf-tas/rekor-redis-7bf4d4b9fb-ww7w4] pod update took 13.202378ms I0327 15:21:45.859865 3944 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-7bf4d4b9fb-ww7w4] addLogicalPort annotation time took 13.234547ms I0327 15:21:45.861104 3944 pods.go:271] [tsf-tas/rekor-redis-7bf4d4b9fb-ww7w4] addLogicalPort took 14.60208ms, libovsdb time 840.387µs I0327 15:21:46.178764 3944 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-4hn7z-bj6gp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:46.178793 3944 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-4hn7z-bj6gp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:46.203970 3944 cni.go:403] [tsf-tas/rekor-redis-7bf4d4b9fb-ww7w4 1a675dbe5e1ae958d7a34b2b7d9048577e324b2c9fb754fcbdf4b8620795b15d network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-7bf4d4b9fb-ww7w4 1a675dbe5e1ae958d7a34b2b7d9048577e324b2c9fb754fcbdf4b8620795b15d network default NAD default] I0327 15:21:46.212267 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-7bf4d4b9fb-ww7w4, hostIfaceName: 1a675dbe5e1ae95, network: default, NAD default, SandboxID: "1a675dbe5e1ae958d7a34b2b7d9048577e324b2c9fb754fcbdf4b8620795b15d", PCI device ID: , UID: "96711bed-0f32-4887-8ed0-8787165d6f13", MAC: 0a:58:0a:82:00:29, IPs: [10.130.0.41/23] I0327 15:21:46.462866 3944 cni.go:424] [tsf-tas/rekor-redis-7bf4d4b9fb-ww7w4 1a675dbe5e1ae958d7a34b2b7d9048577e324b2c9fb754fcbdf4b8620795b15d network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-7bf4d4b9fb-ww7w4 1a675dbe5e1ae958d7a34b2b7d9048577e324b2c9fb754fcbdf4b8620795b15d network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:ea:7f:26:3a:49\",\"name\":\"1a675dbe5e1ae95\"},{\"mac\":\"0a:58:0a:82:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/102b0e14-e58a-4d6a-b5c1-e1a725af6e26\"}],\"ips\":[{\"address\":\"10.130.0.41/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:21:48.171432 3944 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0327 15:21:51.737760 3944 obj_retry.go:464] Detected object tsf/patch-tekton-config-rv5qd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:51.738361 3944 obj_retry.go:464] Detected object tsf/patch-tekton-config-rv5qd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:52.756207 3944 obj_retry.go:458] Detected object tsf/patch-tekton-config-rv5qd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:52.756234 3944 obj_retry.go:458] Detected object tsf/patch-tekton-config-rv5qd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:52.849542 3944 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-cchnn] creating logical port tsf_tsf-tekton-configuration-cchnn for pod on switch ip-10-0-0-80.ec2.internal I0327 15:21:52.849630 3944 kube.go:256] Updating pod tsf/tsf-tekton-configuration-cchnn I0327 15:21:52.859340 3944 pod.go:62] [tsf/tsf-tekton-configuration-cchnn] pod update took 9.731935ms I0327 15:21:52.859363 3944 base_network_controller_pods.go:937] [default/tsf/tsf-tekton-configuration-cchnn] addLogicalPort annotation time took 9.760184ms I0327 15:21:52.860838 3944 pods.go:271] [tsf/tsf-tekton-configuration-cchnn] addLogicalPort took 11.313021ms, libovsdb time 1.074858ms W0327 15:21:53.187319 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0327 15:21:53.204543 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0327 15:21:53.208565 3944 cni.go:403] [tsf/tsf-tekton-configuration-cchnn 1b19a5d66e28720071f9a03620fb37440458abd7976a44b860b6909b89f7e112 network default NAD default] ADD starting CNI request [tsf/tsf-tekton-configuration-cchnn 1b19a5d66e28720071f9a03620fb37440458abd7976a44b860b6909b89f7e112 network default NAD default] I0327 15:21:53.217083 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-cchnn, hostIfaceName: 1b19a5d66e28720, network: default, NAD default, SandboxID: "1b19a5d66e28720071f9a03620fb37440458abd7976a44b860b6909b89f7e112", PCI device ID: , UID: "dd21722a-b67f-443b-91cb-95b05a9939ea", MAC: 0a:58:0a:82:00:2a, IPs: [10.130.0.42/23] I0327 15:21:53.263732 3944 cni.go:424] [tsf/tsf-tekton-configuration-cchnn 1b19a5d66e28720071f9a03620fb37440458abd7976a44b860b6909b89f7e112 network default NAD default] ADD finished CNI request [tsf/tsf-tekton-configuration-cchnn 1b19a5d66e28720071f9a03620fb37440458abd7976a44b860b6909b89f7e112 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:0d:04:75:ea:0e\",\"name\":\"1b19a5d66e28720\"},{\"mac\":\"0a:58:0a:82:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c053e251-bfc1-4306-9522-c369870c5381\"}],\"ips\":[{\"address\":\"10.130.0.42/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:21:58.217912 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0327 15:21:58.231567 3944 cni.go:403] [tsf/tsf-tekton-configuration-cchnn 1b19a5d66e28720071f9a03620fb37440458abd7976a44b860b6909b89f7e112 network default NAD default] DEL starting CNI request [tsf/tsf-tekton-configuration-cchnn 1b19a5d66e28720071f9a03620fb37440458abd7976a44b860b6909b89f7e112 network default NAD default] I0327 15:21:58.311569 3944 cni.go:424] [tsf/tsf-tekton-configuration-cchnn 1b19a5d66e28720071f9a03620fb37440458abd7976a44b860b6909b89f7e112 network default NAD default] DEL finished CNI request [tsf/tsf-tekton-configuration-cchnn 1b19a5d66e28720071f9a03620fb37440458abd7976a44b860b6909b89f7e112 network default NAD default], result "{}", err I0327 15:21:58.355650 3944 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-cchnn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:58.355717 3944 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-cchnn I0327 15:21:58.357070 3944 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-cchnn, ips: 10.130.0.42 I0327 15:21:58.357098 3944 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-cchnn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:21:59.378213 3944 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-cchnn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:59.378245 3944 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-cchnn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:59.415027 3944 obj_retry.go:458] Detected object tsf/patch-tekton-config-rv5qd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:59.415053 3944 obj_retry.go:458] Detected object tsf/patch-tekton-config-rv5qd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:21:59.425187 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-rv5qd I0327 15:21:59.425212 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-rv5qd W0327 15:22:00.215015 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0327 15:22:09.419931 3944 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:09.420506 3944 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:09.436213 3944 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:09.436233 3944 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:09.442983 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0327 15:22:09.443004 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0327 15:22:14.466441 3944 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:14.467008 3944 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:14.485575 3944 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:14.485592 3944 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:14.492439 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0327 15:22:14.492465 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0327 15:22:15.748327 3944 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-4v6df] creating logical port tsf-tpa_create-db-4v6df for pod on switch ip-10-0-0-80.ec2.internal I0327 15:22:15.748413 3944 kube.go:256] Updating pod tsf-tpa/create-db-4v6df I0327 15:22:15.761150 3944 pod.go:62] [tsf-tpa/create-db-4v6df] pod update took 12.757616ms I0327 15:22:15.761174 3944 base_network_controller_pods.go:937] [default/tsf-tpa/create-db-4v6df] addLogicalPort annotation time took 12.787962ms I0327 15:22:15.762617 3944 pods.go:271] [tsf-tpa/create-db-4v6df] addLogicalPort took 14.306395ms, libovsdb time 888.333µs I0327 15:22:16.110245 3944 cni.go:403] [tsf-tpa/create-db-4v6df d7b63480aac9a57710b8f1cb7c86ec99cb19c366fc785c6ede01085c68f429fa network default NAD default] ADD starting CNI request [tsf-tpa/create-db-4v6df d7b63480aac9a57710b8f1cb7c86ec99cb19c366fc785c6ede01085c68f429fa network default NAD default] I0327 15:22:16.119210 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-db-4v6df, hostIfaceName: d7b63480aac9a57, network: default, NAD default, SandboxID: "d7b63480aac9a57710b8f1cb7c86ec99cb19c366fc785c6ede01085c68f429fa", PCI device ID: , UID: "f575dcce-f29b-466b-883e-9d268e60877e", MAC: 0a:58:0a:82:00:2b, IPs: [10.130.0.43/23] I0327 15:22:16.167156 3944 cni.go:424] [tsf-tpa/create-db-4v6df d7b63480aac9a57710b8f1cb7c86ec99cb19c366fc785c6ede01085c68f429fa network default NAD default] ADD finished CNI request [tsf-tpa/create-db-4v6df d7b63480aac9a57710b8f1cb7c86ec99cb19c366fc785c6ede01085c68f429fa network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:e0:76:8d:e2:bf\",\"name\":\"d7b63480aac9a57\"},{\"mac\":\"0a:58:0a:82:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/42bb54ee-cd20-423e-9037-a47010ed5cf6\"}],\"ips\":[{\"address\":\"10.130.0.43/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:22:17.370731 3944 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-pq86c-dt6kd] creating logical port tsf-tas_tuf-repository-init-pq86c-dt6kd for pod on switch ip-10-0-0-80.ec2.internal I0327 15:22:17.370822 3944 kube.go:256] Updating pod tsf-tas/tuf-repository-init-pq86c-dt6kd I0327 15:22:17.382445 3944 pod.go:62] [tsf-tas/tuf-repository-init-pq86c-dt6kd] pod update took 11.648707ms I0327 15:22:17.382471 3944 base_network_controller_pods.go:937] [default/tsf-tas/tuf-repository-init-pq86c-dt6kd] addLogicalPort annotation time took 11.677643ms I0327 15:22:17.383816 3944 pods.go:271] [tsf-tas/tuf-repository-init-pq86c-dt6kd] addLogicalPort took 13.103517ms, libovsdb time 929.128µs I0327 15:22:21.040258 3944 cni.go:403] [tsf-tas/tuf-repository-init-pq86c-dt6kd 23e947e9b00cd95e5c828194b1ff26c58b0e2f01cf4e68962614cec99b871d0c network default NAD default] ADD starting CNI request [tsf-tas/tuf-repository-init-pq86c-dt6kd 23e947e9b00cd95e5c828194b1ff26c58b0e2f01cf4e68962614cec99b871d0c network default NAD default] I0327 15:22:21.048502 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-pq86c-dt6kd, hostIfaceName: 23e947e9b00cd95, network: default, NAD default, SandboxID: "23e947e9b00cd95e5c828194b1ff26c58b0e2f01cf4e68962614cec99b871d0c", PCI device ID: , UID: "3b5aa349-0daf-472c-8b74-2860c6bd98f3", MAC: 0a:58:0a:82:00:2c, IPs: [10.130.0.44/23] I0327 15:22:21.300471 3944 cni.go:424] [tsf-tas/tuf-repository-init-pq86c-dt6kd 23e947e9b00cd95e5c828194b1ff26c58b0e2f01cf4e68962614cec99b871d0c network default NAD default] ADD finished CNI request [tsf-tas/tuf-repository-init-pq86c-dt6kd 23e947e9b00cd95e5c828194b1ff26c58b0e2f01cf4e68962614cec99b871d0c network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:43:67:44:b7:a6\",\"name\":\"23e947e9b00cd95\"},{\"mac\":\"0a:58:0a:82:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ffa1bed0-1633-499c-ba75-e1b328d2c28a\"}],\"ips\":[{\"address\":\"10.130.0.44/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:22:21.347235 3944 cni.go:403] [tsf-tpa/create-db-4v6df d7b63480aac9a57710b8f1cb7c86ec99cb19c366fc785c6ede01085c68f429fa network default NAD default] DEL starting CNI request [tsf-tpa/create-db-4v6df d7b63480aac9a57710b8f1cb7c86ec99cb19c366fc785c6ede01085c68f429fa network default NAD default] I0327 15:22:21.429762 3944 cni.go:424] [tsf-tpa/create-db-4v6df d7b63480aac9a57710b8f1cb7c86ec99cb19c366fc785c6ede01085c68f429fa network default NAD default] DEL finished CNI request [tsf-tpa/create-db-4v6df d7b63480aac9a57710b8f1cb7c86ec99cb19c366fc785c6ede01085c68f429fa network default NAD default], result "{}", err I0327 15:22:21.460136 3944 obj_retry.go:464] Detected object tsf-tpa/create-db-4v6df of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:21.460187 3944 pods.go:176] Deleting pod: tsf-tpa/create-db-4v6df I0327 15:22:21.461581 3944 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-4v6df, ips: 10.130.0.43 I0327 15:22:21.461609 3944 obj_retry.go:464] Detected object tsf-tpa/create-db-4v6df of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:22.484348 3944 obj_retry.go:458] Detected object tsf-tpa/create-db-4v6df of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:22.484375 3944 obj_retry.go:458] Detected object tsf-tpa/create-db-4v6df of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:22.578960 3944 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-h7chx] creating logical port tsf-tpa_migrate-db-h7chx for pod on switch ip-10-0-0-80.ec2.internal I0327 15:22:22.579083 3944 kube.go:256] Updating pod tsf-tpa/migrate-db-h7chx I0327 15:22:22.593533 3944 pod.go:62] [tsf-tpa/migrate-db-h7chx] pod update took 14.492008ms I0327 15:22:22.593562 3944 base_network_controller_pods.go:937] [default/tsf-tpa/migrate-db-h7chx] addLogicalPort annotation time took 14.527557ms I0327 15:22:22.595060 3944 pods.go:271] [tsf-tpa/migrate-db-h7chx] addLogicalPort took 16.123348ms, libovsdb time 953.47µs I0327 15:22:22.940864 3944 cni.go:403] [tsf-tpa/migrate-db-h7chx 78a4cafad966298ba4d94b93cac539af5bfe7c930a72dcdcec14a57735c7e9a2 network default NAD default] ADD starting CNI request [tsf-tpa/migrate-db-h7chx 78a4cafad966298ba4d94b93cac539af5bfe7c930a72dcdcec14a57735c7e9a2 network default NAD default] I0327 15:22:22.949435 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-h7chx, hostIfaceName: 78a4cafad966298, network: default, NAD default, SandboxID: "78a4cafad966298ba4d94b93cac539af5bfe7c930a72dcdcec14a57735c7e9a2", PCI device ID: , UID: "b8b7b8ff-dfca-4929-b3ca-88103cfd4773", MAC: 0a:58:0a:82:00:2d, IPs: [10.130.0.45/23] I0327 15:22:23.002889 3944 cni.go:424] [tsf-tpa/migrate-db-h7chx 78a4cafad966298ba4d94b93cac539af5bfe7c930a72dcdcec14a57735c7e9a2 network default NAD default] ADD finished CNI request [tsf-tpa/migrate-db-h7chx 78a4cafad966298ba4d94b93cac539af5bfe7c930a72dcdcec14a57735c7e9a2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:87:ee:3a:d7:d9\",\"name\":\"78a4cafad966298\"},{\"mac\":\"0a:58:0a:82:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0f1c8bc3-d0f7-494d-8a2a-964b956634fe\"}],\"ips\":[{\"address\":\"10.130.0.45/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:22:24.368239 3944 cni.go:403] [tsf-tas/tuf-repository-init-pq86c-dt6kd 23e947e9b00cd95e5c828194b1ff26c58b0e2f01cf4e68962614cec99b871d0c network default NAD default] DEL starting CNI request [tsf-tas/tuf-repository-init-pq86c-dt6kd 23e947e9b00cd95e5c828194b1ff26c58b0e2f01cf4e68962614cec99b871d0c network default NAD default] I0327 15:22:24.446767 3944 cni.go:424] [tsf-tas/tuf-repository-init-pq86c-dt6kd 23e947e9b00cd95e5c828194b1ff26c58b0e2f01cf4e68962614cec99b871d0c network default NAD default] DEL finished CNI request [tsf-tas/tuf-repository-init-pq86c-dt6kd 23e947e9b00cd95e5c828194b1ff26c58b0e2f01cf4e68962614cec99b871d0c network default NAD default], result "{}", err I0327 15:22:24.493395 3944 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pq86c-dt6kd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:24.493433 3944 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-pq86c-dt6kd I0327 15:22:24.494751 3944 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-pq86c-dt6kd, ips: 10.130.0.44 I0327 15:22:24.494780 3944 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-pq86c-dt6kd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.377013 3944 cni.go:403] [tsf-tpa/migrate-db-h7chx 78a4cafad966298ba4d94b93cac539af5bfe7c930a72dcdcec14a57735c7e9a2 network default NAD default] DEL starting CNI request [tsf-tpa/migrate-db-h7chx 78a4cafad966298ba4d94b93cac539af5bfe7c930a72dcdcec14a57735c7e9a2 network default NAD default] I0327 15:22:25.379288 3944 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pq86c-dt6kd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.379318 3944 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-pq86c-dt6kd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.469377 3944 cni.go:424] [tsf-tpa/migrate-db-h7chx 78a4cafad966298ba4d94b93cac539af5bfe7c930a72dcdcec14a57735c7e9a2 network default NAD default] DEL finished CNI request [tsf-tpa/migrate-db-h7chx 78a4cafad966298ba4d94b93cac539af5bfe7c930a72dcdcec14a57735c7e9a2 network default NAD default], result "{}", err I0327 15:22:25.514978 3944 obj_retry.go:464] Detected object tsf-tpa/migrate-db-h7chx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.515016 3944 pods.go:176] Deleting pod: tsf-tpa/migrate-db-h7chx I0327 15:22:25.516490 3944 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-h7chx, ips: 10.130.0.45 I0327 15:22:25.516525 3944 obj_retry.go:464] Detected object tsf-tpa/migrate-db-h7chx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:25.609497 3944 obj_retry.go:458] Detected object tsf-tpa/migrate-db-h7chx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:25.609519 3944 obj_retry.go:458] Detected object tsf-tpa/migrate-db-h7chx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:28.736922 3944 base_network_controller_pods.go:487] [default/tsf-tpa/server-594cf5b874-6hc9m] creating logical port tsf-tpa_server-594cf5b874-6hc9m for pod on switch ip-10-0-0-80.ec2.internal I0327 15:22:28.737021 3944 kube.go:256] Updating pod tsf-tpa/server-594cf5b874-6hc9m I0327 15:22:28.751411 3944 base_network_controller_pods.go:487] [default/tsf-tpa/importer-74cd48f5c7-7d8tp] creating logical port tsf-tpa_importer-74cd48f5c7-7d8tp for pod on switch ip-10-0-0-80.ec2.internal I0327 15:22:28.751560 3944 kube.go:256] Updating pod tsf-tpa/importer-74cd48f5c7-7d8tp I0327 15:22:28.754541 3944 pod.go:62] [tsf-tpa/server-594cf5b874-6hc9m] pod update took 17.547772ms I0327 15:22:28.754565 3944 base_network_controller_pods.go:937] [default/tsf-tpa/server-594cf5b874-6hc9m] addLogicalPort annotation time took 17.57618ms I0327 15:22:28.755848 3944 pods.go:271] [tsf-tpa/server-594cf5b874-6hc9m] addLogicalPort took 18.945574ms, libovsdb time 843.565µs I0327 15:22:28.767317 3944 pod.go:62] [tsf-tpa/importer-74cd48f5c7-7d8tp] pod update took 15.822856ms I0327 15:22:28.767337 3944 base_network_controller_pods.go:937] [default/tsf-tpa/importer-74cd48f5c7-7d8tp] addLogicalPort annotation time took 15.847425ms I0327 15:22:28.768847 3944 pods.go:271] [tsf-tpa/importer-74cd48f5c7-7d8tp] addLogicalPort took 17.457012ms, libovsdb time 646.676µs I0327 15:22:29.456962 3944 base_network_controller_pods.go:487] [default/tsf-tas/tuf-bc5768c88-85frg] creating logical port tsf-tas_tuf-bc5768c88-85frg for pod on switch ip-10-0-0-80.ec2.internal I0327 15:22:29.457059 3944 kube.go:256] Updating pod tsf-tas/tuf-bc5768c88-85frg I0327 15:22:29.470445 3944 pod.go:62] [tsf-tas/tuf-bc5768c88-85frg] pod update took 13.416658ms I0327 15:22:29.470618 3944 base_network_controller_pods.go:937] [default/tsf-tas/tuf-bc5768c88-85frg] addLogicalPort annotation time took 13.588167ms I0327 15:22:29.475956 3944 pods.go:271] [tsf-tas/tuf-bc5768c88-85frg] addLogicalPort took 19.010416ms, libovsdb time 2.915908ms I0327 15:22:31.036870 3944 cni.go:403] [tsf-tas/tuf-bc5768c88-85frg 01ffb93790bd2cb5489a24b6cacb28ca4e406653732f67a9ef79189ad4aead06 network default NAD default] ADD starting CNI request [tsf-tas/tuf-bc5768c88-85frg 01ffb93790bd2cb5489a24b6cacb28ca4e406653732f67a9ef79189ad4aead06 network default NAD default] I0327 15:22:31.045171 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-bc5768c88-85frg, hostIfaceName: 01ffb93790bd2cb, network: default, NAD default, SandboxID: "01ffb93790bd2cb5489a24b6cacb28ca4e406653732f67a9ef79189ad4aead06", PCI device ID: , UID: "47e3c386-154c-4f51-9253-fdc23024e6e0", MAC: 0a:58:0a:82:00:30, IPs: [10.130.0.48/23] I0327 15:22:31.091646 3944 cni.go:424] [tsf-tas/tuf-bc5768c88-85frg 01ffb93790bd2cb5489a24b6cacb28ca4e406653732f67a9ef79189ad4aead06 network default NAD default] ADD finished CNI request [tsf-tas/tuf-bc5768c88-85frg 01ffb93790bd2cb5489a24b6cacb28ca4e406653732f67a9ef79189ad4aead06 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:72:80:3b:7e:de\",\"name\":\"01ffb93790bd2cb\"},{\"mac\":\"0a:58:0a:82:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a2d1f72b-8058-4bfa-9848-33434f3f929b\"}],\"ips\":[{\"address\":\"10.130.0.48/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:22:31.291773 3944 obj_retry.go:464] Detected object tsf-tpa/create-importers-9h4cb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:31.292437 3944 obj_retry.go:464] Detected object tsf-tpa/create-importers-9h4cb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:22:32.317138 3944 obj_retry.go:458] Detected object tsf-tpa/create-importers-9h4cb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:32.317166 3944 obj_retry.go:458] Detected object tsf-tpa/create-importers-9h4cb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:22:34.200137 3944 cni.go:403] [tsf-tpa/server-594cf5b874-6hc9m 37b5389a48839d6d514b22b73b3997c711ecec4a795e4d857379273a6898936b network default NAD default] ADD starting CNI request [tsf-tpa/server-594cf5b874-6hc9m 37b5389a48839d6d514b22b73b3997c711ecec4a795e4d857379273a6898936b network default NAD default] I0327 15:22:34.208782 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: server-594cf5b874-6hc9m, hostIfaceName: 37b5389a48839d6, network: default, NAD default, SandboxID: "37b5389a48839d6d514b22b73b3997c711ecec4a795e4d857379273a6898936b", PCI device ID: , UID: "cfa922eb-1a06-4a66-9b53-7e4153a98c5d", MAC: 0a:58:0a:82:00:2e, IPs: [10.130.0.46/23] I0327 15:22:34.213572 3944 cni.go:403] [tsf-tpa/importer-74cd48f5c7-7d8tp 9f9b3b77be0038c24ae6785dce086ca94536bd17a4f0c778207ee66e2d22df5f network default NAD default] ADD starting CNI request [tsf-tpa/importer-74cd48f5c7-7d8tp 9f9b3b77be0038c24ae6785dce086ca94536bd17a4f0c778207ee66e2d22df5f network default NAD default] I0327 15:22:34.221739 3944 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: importer-74cd48f5c7-7d8tp, hostIfaceName: 9f9b3b77be0038c, network: default, NAD default, SandboxID: "9f9b3b77be0038c24ae6785dce086ca94536bd17a4f0c778207ee66e2d22df5f", PCI device ID: , UID: "40bb1107-3df0-4d44-89e3-9ce006c72024", MAC: 0a:58:0a:82:00:2f, IPs: [10.130.0.47/23] I0327 15:22:34.259473 3944 cni.go:424] [tsf-tpa/server-594cf5b874-6hc9m 37b5389a48839d6d514b22b73b3997c711ecec4a795e4d857379273a6898936b network default NAD default] ADD finished CNI request [tsf-tpa/server-594cf5b874-6hc9m 37b5389a48839d6d514b22b73b3997c711ecec4a795e4d857379273a6898936b network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:b6:6b:5d:3f:5f\",\"name\":\"37b5389a48839d6\"},{\"mac\":\"0a:58:0a:82:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c1c87188-37be-49f6-b360-83e29d7aeecb\"}],\"ips\":[{\"address\":\"10.130.0.46/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err I0327 15:22:34.280853 3944 cni.go:424] [tsf-tpa/importer-74cd48f5c7-7d8tp 9f9b3b77be0038c24ae6785dce086ca94536bd17a4f0c778207ee66e2d22df5f network default NAD default] ADD finished CNI request [tsf-tpa/importer-74cd48f5c7-7d8tp 9f9b3b77be0038c24ae6785dce086ca94536bd17a4f0c778207ee66e2d22df5f network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:f3:06:59:33:73\",\"name\":\"9f9b3b77be0038c\"},{\"mac\":\"0a:58:0a:82:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7dddbf93-1ca5-4b0b-82f4-0047f884a136\"}],\"ips\":[{\"address\":\"10.130.0.47/23\",\"gateway\":\"10.130.0.1\",\"interface\":1}]}", err W0327 15:22:34.417340 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0327 15:22:34.443192 3944 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0327 15:22:45.434905 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server W0327 15:22:45.471102 3944 util.go:959] Failed to build node endpoints for node ip-10-0-0-80.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf I0327 15:23:02.778386 3944 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:02.778995 3944 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:02.816479 3944 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:02.816509 3944 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:02.823359 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0327 15:23:02.823377 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0327 15:23:03.623629 3944 namespace.go:98] [image-controller] adding namespace I0327 15:23:03.624625 3944 namespace.go:102] [image-controller] adding namespace took 975.748µs I0327 15:23:03.627279 3944 namespace.go:98] [integration-service] adding namespace I0327 15:23:03.627329 3944 namespace.go:98] [build-service] adding namespace I0327 15:23:03.627984 3944 namespace.go:102] [integration-service] adding namespace took 684.691µs I0327 15:23:03.628566 3944 namespace.go:102] [build-service] adding namespace took 1.220016ms I0327 15:23:03.678218 3944 namespace.go:142] [image-controller] updating namespace I0327 15:23:03.716248 3944 namespace.go:142] [image-controller] updating namespace I0327 15:23:03.720707 3944 namespace.go:142] [integration-service] updating namespace I0327 15:23:03.761565 3944 namespace.go:142] [build-service] updating namespace I0327 15:23:03.766505 3944 namespace.go:142] [integration-service] updating namespace I0327 15:23:03.794235 3944 namespace.go:142] [build-service] updating namespace I0327 15:23:03.830572 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0327 15:23:03.891187 3944 namespace.go:142] [build-service] updating namespace I0327 15:23:03.927133 3944 namespace.go:142] [integration-service] updating namespace I0327 15:23:03.988211 3944 namespace.go:98] [release-service] adding namespace I0327 15:23:03.989098 3944 namespace.go:102] [release-service] adding namespace took 870.708µs I0327 15:23:04.090139 3944 namespace.go:142] [release-service] updating namespace I0327 15:23:04.090276 3944 namespace.go:142] [konflux-ui] updating namespace I0327 15:23:04.138314 3944 namespace.go:142] [release-service] updating namespace I0327 15:23:04.229820 3944 namespace.go:98] [konflux-info] adding namespace I0327 15:23:04.230703 3944 namespace.go:102] [konflux-info] adding namespace took 862.432µs I0327 15:23:04.337880 3944 namespace.go:98] [namespace-lister] adding namespace I0327 15:23:04.338047 3944 namespace.go:142] [konflux-info] updating namespace I0327 15:23:04.338735 3944 namespace.go:102] [namespace-lister] adding namespace took 832.892µs I0327 15:23:04.381155 3944 namespace.go:98] [enterprise-contract-service] adding namespace I0327 15:23:04.381855 3944 namespace.go:102] [enterprise-contract-service] adding namespace took 682.181µs I0327 15:23:04.419725 3944 namespace.go:142] [image-controller] updating namespace I0327 15:23:04.522239 3944 namespace.go:142] [konflux-info] updating namespace I0327 15:23:04.522249 3944 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:04.641085 3944 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:04.718117 3944 namespace.go:98] [default-tenant] adding namespace I0327 15:23:04.718220 3944 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:04.718989 3944 namespace.go:102] [default-tenant] adding namespace took 849.39µs I0327 15:23:04.807022 3944 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:04.824019 3944 namespace.go:98] [segment-bridge] adding namespace I0327 15:23:04.824789 3944 namespace.go:102] [segment-bridge] adding namespace took 751.772µs I0327 15:23:04.881247 3944 namespace.go:142] [default-tenant] updating namespace I0327 15:23:04.960597 3944 namespace.go:142] [default-tenant] updating namespace I0327 15:23:04.966185 3944 namespace.go:142] [segment-bridge] updating namespace I0327 15:23:05.022188 3944 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0327 15:23:05.022841 3944 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0327 15:23:05.022861 3944 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0327 15:23:05.025400 3944 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0327 15:23:05.031117 3944 namespace.go:142] [segment-bridge] updating namespace I0327 15:23:05.076253 3944 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0327 15:23:05.076277 3944 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0327 15:23:05.077113 3944 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0327 15:23:05.318156 3944 namespace.go:142] [build-service] updating namespace I0327 15:23:05.708145 3944 namespace.go:142] [image-controller] updating namespace I0327 15:23:06.221421 3944 namespace.go:142] [integration-service] updating namespace I0327 15:23:06.606027 3944 namespace.go:142] [konflux-info] updating namespace I0327 15:23:06.930147 3944 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:07.200773 3944 namespace.go:142] [release-service] updating namespace I0327 15:23:07.343324 3944 namespace.go:142] [build-service] updating namespace I0327 15:23:07.364284 3944 namespace.go:142] [image-controller] updating namespace I0327 15:23:07.398025 3944 namespace.go:142] [integration-service] updating namespace I0327 15:23:07.460596 3944 namespace.go:142] [konflux-info] updating namespace I0327 15:23:07.496053 3944 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:07.513213 3944 namespace.go:142] [release-service] updating namespace I0327 15:23:07.591334 3944 namespace.go:142] [build-service] updating namespace I0327 15:23:07.722951 3944 namespace.go:142] [image-controller] updating namespace I0327 15:23:07.837591 3944 namespace.go:142] [integration-service] updating namespace I0327 15:23:07.851794 3944 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-9rx2b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:07.852359 3944 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-9rx2b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:07.923918 3944 namespace.go:142] [konflux-info] updating namespace I0327 15:23:08.080310 3944 namespace.go:142] [namespace-lister] updating namespace I0327 15:23:08.185242 3944 namespace.go:142] [release-service] updating namespace I0327 15:23:08.852030 3944 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-9rx2b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:08.852060 3944 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-9rx2b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:09.329740 3944 namespace.go:142] [default-tenant] updating namespace I0327 15:23:09.536703 3944 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:09.731992 3944 namespace.go:142] [segment-bridge] updating namespace I0327 15:23:09.852461 3944 namespace.go:142] [default-tenant] updating namespace I0327 15:23:09.868071 3944 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:09.881159 3944 namespace.go:142] [segment-bridge] updating namespace I0327 15:23:09.962934 3944 namespace.go:142] [default-tenant] updating namespace I0327 15:23:10.068077 3944 namespace.go:142] [enterprise-contract-service] updating namespace I0327 15:23:10.168541 3944 namespace.go:142] [segment-bridge] updating namespace W0327 15:23:10.374462 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0327 15:23:12.388222 3944 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0327 15:23:12.416083 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0327 15:23:15.736741 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=80 W0327 15:23:16.745989 3944 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:23:16.748479 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:23:16.763749 3944 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0327 15:23:16.764272 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0327 15:23:19.443043 3944 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 15:23:19.443060 3944 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0327 15:23:19.461846 3944 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0327 15:23:20.448223 3944 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0327 15:23:20.453000 3944 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service I0327 15:23:20.748799 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=514 W0327 15:23:23.821224 3944 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0327 15:23:23.821242 3944 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0327 15:23:28.084104 3944 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-zpmqr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:28.084706 3944 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-zpmqr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:23:28.828970 3944 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-zpmqr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:28.828992 3944 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-zpmqr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:23:28.831798 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-zpmqr I0327 15:23:28.831824 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-zpmqr I0327 15:24:00.730570 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=114 I0327 15:24:22.840211 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0327 15:24:32.349116 3944 obj_retry.go:464] Detected object konflux-ui/proxy-658b44b84-npv7x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:32.349690 3944 obj_retry.go:464] Detected object konflux-ui/proxy-658b44b84-npv7x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:32.864532 3944 obj_retry.go:458] Detected object konflux-ui/proxy-658b44b84-npv7x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:32.864556 3944 obj_retry.go:458] Detected object konflux-ui/proxy-658b44b84-npv7x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:32.872316 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-658b44b84-npv7x I0327 15:24:32.872348 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-658b44b84-npv7x I0327 15:24:41.416223 3944 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:41.416825 3944 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:24:41.435008 3944 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:41.435032 3944 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:24:41.440163 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0327 15:24:41.440184 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0327 15:24:59.855395 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0327 15:25:09.150532 3944 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:25:09.151183 3944 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:25:10.066690 3944 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:25:10.066714 3944 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:25:10.075695 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 I0327 15:25:10.075717 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-mht92 I0327 15:25:23.811288 3944 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-nwslp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:25:23.811920 3944 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-nwslp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:25:24.151536 3944 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-nwslp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:25:24.151558 3944 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-nwslp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:25:24.158366 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-nwslp I0327 15:25:24.158389 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-nwslp I0327 15:26:02.817754 3944 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0327 15:26:34.852142 3944 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0327 15:26:52.736031 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=117 I0327 15:26:58.859268 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0327 15:27:25.735852 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0327 15:27:36.730389 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=57 I0327 15:28:07.846027 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0327 15:28:54.750658 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=52 I0327 15:29:39.819242 3944 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0327 15:29:46.824006 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0327 15:29:53.835454 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0327 15:30:03.857746 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0327 15:30:04.322904 3944 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-qw9p6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0327 15:30:04.323525 3944 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-qw9p6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0327 15:30:05.194244 3944 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-qw9p6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:30:05.194291 3944 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29577090-qw9p6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:31:47.842584 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0327 15:31:57.820135 3944 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0327 15:32:22.046461 3944 obj_retry.go:458] Detected object tsf-tpa/create-db-4v6df of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:22.046485 3944 obj_retry.go:458] Detected object tsf-tpa/create-db-4v6df of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:22.048923 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-4v6df I0327 15:32:22.049009 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-4v6df I0327 15:32:25.025860 3944 obj_retry.go:458] Detected object tsf-tpa/migrate-db-h7chx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:25.025905 3944 obj_retry.go:458] Detected object tsf-tpa/migrate-db-h7chx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:25.029937 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-h7chx I0327 15:32:25.029964 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-h7chx I0327 15:32:28.861340 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=5 I0327 15:32:32.045404 3944 obj_retry.go:458] Detected object tsf-tpa/create-importers-9h4cb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:32.045457 3944 obj_retry.go:458] Detected object tsf-tpa/create-importers-9h4cb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0327 15:32:32.047534 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-9h4cb I0327 15:32:32.047563 3944 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-9h4cb I0327 15:33:05.733599 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=20 I0327 15:33:07.739262 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0327 15:33:24.858027 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0327 15:33:56.854572 3944 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0327 15:33:59.848713 3944 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0327 15:35:14.821011 3944 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0327 15:35:32.738085 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=9 I0327 15:36:08.733001 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=44 I0327 15:36:42.737759 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0327 15:37:28.754135 3944 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=25