+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-243.ec2.internal ++ [[ -n ip-10-0-1-243.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-243.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0331 03:48:49.373576252 - disable conntrack on geneve port' I0331 03:48:49.373576252 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0331 03:48:49.390536935 - starting ovnkube-node + echo 'I0331 03:48:49.390536935 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-243.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-243.ec2.internal --init-node ip-10-0-1-243.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-243.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0331 03:48:49.428083 4042 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0331 03:48:49.428124 4042 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-8259435a9f.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0331 03:48:49.429898 4042 kube.go:419] Waiting for certificate I0331 03:48:49.429934 4042 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0331 03:48:49.429972 4042 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0331 03:48:49.430792 4042 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0331 03:48:49.458690 4042 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:48:49.458705 4042 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:48:49.461164 4042 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:48:49.463551 4042 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rv7rc" I0331 03:48:49.473093 4042 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-rv7rc" I0331 03:48:49.473166 4042 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:48:50.430806 4042 kube.go:426] Certificate found I0331 03:48:50.431363 4042 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0331 03:48:50.432249 4042 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0331 03:48:50.478460 4042 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-01 03:43:49 +0000 UTC" deadline="2026-04-01 01:03:43.750620984 +0000 UTC" I0331 03:48:50.478516 4042 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="21h14m53.272133871s" I0331 03:48:50.496806 4042 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0331 03:48:50.509383 4042 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0331 03:48:50.512796 4042 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0331 03:48:50.512869 4042 factory.go:531] Starting watch factory I0331 03:48:50.512971 4042 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.512967 4042 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.512990 4042 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.512996 4042 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.513001 4042 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.512971 4042 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.513023 4042 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.512975 4042 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.513089 4042 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.513008 4042 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.513603 4042 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.513658 4042 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.519154 4042 controller_manager.go:368] Starting the ovnkube controller I0331 03:48:50.519168 4042 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-243.ec2.internal I0331 03:48:50.519223 4042 controller_manager.go:393] NBDB zone sync took: 45.909µs I0331 03:48:50.519234 4042 factory.go:531] Starting watch factory I0331 03:48:50.530489 4042 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.530875 4042 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.533146 4042 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.534477 4042 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.534713 4042 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0331 03:48:50.535612 4042 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.536165 4042 ovn_northd.go:121] Registering OVN northd metrics I0331 03:48:50.538378 4042 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0331 03:48:50.538481 4042 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0331 03:48:50.538928 4042 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:48:50.555027 4042 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0331 03:48:50.618166 4042 factory.go:1890] *v1.Service informer cache synced successfully I0331 03:48:50.618188 4042 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0331 03:48:50.618192 4042 factory.go:1890] *v1.Pod informer cache synced successfully I0331 03:48:50.618199 4042 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0331 03:48:50.618208 4042 factory.go:1890] *v1.Namespace informer cache synced successfully I0331 03:48:50.618211 4042 factory.go:1890] *v1.Node informer cache synced successfully I0331 03:48:50.618296 4042 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:48:50.618302 4042 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:48:50.618315 4042 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:48:50.618321 4042 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:48:50.620762 4042 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:48:50.620950 4042 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:48:50.623287 4042 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0331 03:48:50.623298 4042 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0331 03:48:50.623349 4042 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.623359 4042 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.625462 4042 factory.go:1890] *v1.Node informer cache synced successfully I0331 03:48:50.625473 4042 factory.go:1890] *v1.Service informer cache synced successfully I0331 03:48:50.625477 4042 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0331 03:48:50.625480 4042 factory.go:1890] *v1.Pod informer cache synced successfully I0331 03:48:50.625483 4042 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0331 03:48:50.625486 4042 factory.go:1890] *v1.Namespace informer cache synced successfully I0331 03:48:50.625709 4042 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.628666 4042 factory.go:1890] *v1.EgressIP informer cache synced successfully I0331 03:48:50.628734 4042 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.628750 4042 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.630603 4042 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0331 03:48:50.630614 4042 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0331 03:48:50.630774 4042 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.634723 4042 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0331 03:48:50.634778 4042 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.634790 4042 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.636160 4042 factory.go:1890] *v1.EgressIP informer cache synced successfully I0331 03:48:50.636867 4042 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.640735 4042 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0331 03:48:50.640800 4042 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.640817 4042 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.642125 4042 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0331 03:48:50.642878 4042 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.646675 4042 factory.go:1890] *v1.EgressService informer cache synced successfully I0331 03:48:50.646723 4042 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.646735 4042 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.647817 4042 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0331 03:48:50.648691 4042 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.652442 4042 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0331 03:48:50.652593 4042 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:48:50.652603 4042 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:48:50.653132 4042 factory.go:1890] *v1.EgressService informer cache synced successfully I0331 03:48:50.654609 4042 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:48:50.658359 4042 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0331 03:48:50.658372 4042 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0331 03:48:50.658437 4042 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.658450 4042 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.658514 4042 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.658536 4042 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.660716 4042 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.660925 4042 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:48:50.663939 4042 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0331 03:48:50.663949 4042 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0331 03:48:50.663951 4042 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0331 03:48:50.663961 4042 factory.go:636] Watch Factory start up complete, took: 151.09528ms I0331 03:48:50.664023 4042 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0331 03:48:50.669511 4042 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0331 03:48:50.669528 4042 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0331 03:48:50.669538 4042 factory.go:636] Watch Factory start up complete, took: 150.301945ms I0331 03:48:50.669549 4042 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-1-243.ec2.internal" zone I0331 03:48:50.706115 4042 default_node_network_controller.go:709] Initializing the default node network controller I0331 03:48:50.757716 4042 udn_isolation.go:101] Starting UDN host isolation manager I0331 03:48:50.763892 4042 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0331 03:48:50.861405 4042 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0331 03:48:50.861471 4042 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0331 03:48:50.861484 4042 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0331 03:48:50.893322 4042 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0331 03:48:50.893382 4042 default_node_network_controller.go:808] Node ip-10-0-1-243.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0331 03:48:50.898082 4042 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.243"] k8s.ovn.org/zone-name:ip-10-0-1-243.ec2.internal] on node ip-10-0-1-243.ec2.internal I0331 03:48:50.918360 4042 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0331 03:48:50.920599 4042 controller_manager.go:427] Waiting for node in zone sync took: 251.037477ms I0331 03:48:50.923972 4042 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0331 03:48:50.924120 4042 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0331 03:48:50.924266 4042 gateway_init.go:263] Preparing Gateway I0331 03:48:50.924282 4042 gateway_shared_intf.go:1658] Creating new gateway I0331 03:48:50.925450 4042 controller_manager.go:608] Switching to transit router for layer2 networks I0331 03:48:50.925464 4042 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-243.ec2.internal I0331 03:48:50.941180 4042 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0331 03:48:50.946759 4042 controller_manager.go:288] SCTP support detected in OVN I0331 03:48:50.947382 4042 services_controller.go:69] Creating services controller for network=default I0331 03:48:50.947471 4042 udn_enabled_svc.go:80] Waiting for service informer to sync I0331 03:48:50.947592 4042 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0331 03:48:50.947629 4042 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0331 03:48:50.947657 4042 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0331 03:48:50.947670 4042 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0331 03:48:50.947921 4042 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0331 03:48:50.948024 4042 udn_enabled_svc.go:95] Performing full resync I0331 03:48:50.948043 4042 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0331 03:48:50.948080 4042 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 29.148µs I0331 03:48:50.948090 4042 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 48.083µs I0331 03:48:50.948098 4042 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0331 03:48:50.948105 4042 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0331 03:48:50.948119 4042 default_network_controller.go:321] Starting the default network controller I0331 03:48:50.948145 4042 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 121.729µs I0331 03:48:50.948233 4042 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 22.603µs I0331 03:48:50.948422 4042 udn_enabled_svc.go:99] Waiting for handler to sync I0331 03:48:50.948435 4042 udn_enabled_svc.go:104] Starting worker I0331 03:48:50.956894 4042 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0331 03:48:50.956956 4042 repair.go:33] Syncing exgw routes took 42.954µs I0331 03:48:50.956972 4042 default_network_controller.go:407] Starting all the Watchers... I0331 03:48:50.957105 4042 namespace.go:98] [openshift-console-user-settings] adding namespace I0331 03:48:50.957135 4042 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0331 03:48:50.957142 4042 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0331 03:48:50.957147 4042 namespace.go:98] [openshift-authentication-operator] adding namespace I0331 03:48:50.957151 4042 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0331 03:48:50.957173 4042 namespace.go:98] [openshift-etcd] adding namespace I0331 03:48:50.957177 4042 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0331 03:48:50.957186 4042 namespace.go:98] [openshift-backplane-srep] adding namespace I0331 03:48:50.957217 4042 namespace.go:98] [kube-system] adding namespace I0331 03:48:50.957231 4042 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0331 03:48:50.957238 4042 namespace.go:98] [openshift-multus] adding namespace I0331 03:48:50.957249 4042 namespace.go:98] [openshift-ingress] adding namespace I0331 03:48:50.957258 4042 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0331 03:48:50.957103 4042 namespace.go:98] [dedicated-admin] adding namespace I0331 03:48:50.957266 4042 namespace.go:98] [openshift-backplane-tam] adding namespace I0331 03:48:50.957801 4042 namespace.go:102] [openshift-console-user-settings] adding namespace took 676.283µs I0331 03:48:50.957816 4042 namespace.go:98] [openshift-authentication] adding namespace I0331 03:48:50.958274 4042 namespace.go:102] [openshift-authentication] adding namespace took 452.735µs I0331 03:48:50.958287 4042 namespace.go:98] [openshift-image-registry] adding namespace I0331 03:48:50.958795 4042 namespace.go:102] [openshift-image-registry] adding namespace took 502.042µs I0331 03:48:50.958812 4042 namespace.go:98] [openshift-console-operator] adding namespace I0331 03:48:50.959423 4042 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 2.275381ms I0331 03:48:50.959438 4042 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0331 03:48:50.960426 4042 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 3.269878ms I0331 03:48:50.960447 4042 namespace.go:98] [openshift-route-controller-manager] adding namespace I0331 03:48:50.961033 4042 namespace.go:102] [openshift-authentication-operator] adding namespace took 3.877948ms I0331 03:48:50.961050 4042 namespace.go:98] [openshift-backplane-mobb] adding namespace I0331 03:48:50.961651 4042 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 4.488263ms I0331 03:48:50.961671 4042 namespace.go:98] [openshift-config] adding namespace I0331 03:48:50.962252 4042 namespace.go:102] [openshift-etcd] adding namespace took 5.068981ms I0331 03:48:50.962268 4042 namespace.go:98] [openshift-apiserver-operator] adding namespace I0331 03:48:50.962856 4042 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 5.664653ms I0331 03:48:50.962875 4042 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0331 03:48:50.963365 4042 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.166644ms I0331 03:48:50.963388 4042 namespace.go:98] [openshift-backplane-csm] adding namespace W0331 03:48:50.963390 4042 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-wp5mx: pod kube-system/global-pull-secret-syncer-wp5mx: no pod IPs found W0331 03:48:50.963403 4042 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-r9vx7: pod kube-system/global-pull-secret-syncer-r9vx7: no pod IPs found I0331 03:48:50.963987 4042 namespace.go:102] [kube-system] adding namespace took 6.755207ms I0331 03:48:50.964004 4042 namespace.go:98] [openshift-insights] adding namespace W0331 03:48:50.964545 4042 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-46hqv: pod openshift-multus/network-metrics-daemon-46hqv: no pod IPs found W0331 03:48:50.964557 4042 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-9rdwh: pod openshift-multus/network-metrics-daemon-9rdwh: no pod IPs found W0331 03:48:50.964563 4042 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-lwlnf: pod openshift-multus/network-metrics-daemon-lwlnf: no pod IPs found I0331 03:48:50.964581 4042 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.339484ms I0331 03:48:50.964593 4042 namespace.go:98] [openshift-network-console] adding namespace I0331 03:48:50.965153 4042 namespace.go:102] [openshift-multus] adding namespace took 7.906468ms I0331 03:48:50.965166 4042 namespace.go:98] [open-cluster-management-2pcf4qvmb8c8ij76t9r5ocbboorpi5e4] adding namespace I0331 03:48:50.965770 4042 namespace.go:102] [openshift-ingress] adding namespace took 8.513153ms I0331 03:48:50.965785 4042 namespace.go:98] [openshift-customer-monitoring] adding namespace I0331 03:48:50.966290 4042 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 9.023888ms I0331 03:48:50.966304 4042 namespace.go:98] [openshift-network-operator] adding namespace I0331 03:48:50.966747 4042 namespace.go:102] [dedicated-admin] adding namespace took 9.477917ms I0331 03:48:50.966759 4042 namespace.go:98] [openshift-backplane-cee] adding namespace I0331 03:48:50.967339 4042 namespace.go:102] [openshift-backplane-tam] adding namespace took 10.064705ms I0331 03:48:50.967353 4042 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0331 03:48:50.967907 4042 namespace.go:102] [openshift-console-operator] adding namespace took 9.086594ms I0331 03:48:50.967930 4042 namespace.go:98] [openshift-machine-config-operator] adding namespace I0331 03:48:50.968364 4042 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 8.919412ms I0331 03:48:50.968380 4042 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0331 03:48:50.968893 4042 namespace.go:102] [openshift-route-controller-manager] adding namespace took 8.440786ms I0331 03:48:50.968907 4042 namespace.go:98] [openshift-host-network] adding namespace I0331 03:48:50.969347 4042 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.286106ms I0331 03:48:50.969369 4042 namespace.go:98] [openshift-monitoring] adding namespace I0331 03:48:50.969853 4042 namespace.go:102] [openshift-config] adding namespace took 8.173989ms I0331 03:48:50.969877 4042 namespace.go:98] [openshift-operators-redhat] adding namespace I0331 03:48:50.970427 4042 namespace.go:102] [openshift-apiserver-operator] adding namespace took 8.151746ms I0331 03:48:50.970459 4042 namespace.go:98] [openshift-console] adding namespace I0331 03:48:50.971264 4042 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 8.379444ms I0331 03:48:50.971285 4042 namespace.go:98] [openshift-ingress-operator] adding namespace I0331 03:48:50.971799 4042 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.404197ms I0331 03:48:50.971828 4042 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0331 03:48:50.972330 4042 namespace.go:102] [openshift-insights] adding namespace took 8.321067ms I0331 03:48:50.972343 4042 namespace.go:98] [openshift-kube-apiserver] adding namespace I0331 03:48:50.972850 4042 namespace.go:102] [openshift-network-console] adding namespace took 8.247863ms I0331 03:48:50.972864 4042 namespace.go:98] [openshift-network-node-identity] adding namespace I0331 03:48:50.973436 4042 namespace.go:102] [open-cluster-management-2pcf4qvmb8c8ij76t9r5ocbboorpi5e4] adding namespace took 8.263635ms I0331 03:48:50.973452 4042 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0331 03:48:50.973937 4042 namespace.go:102] [openshift-customer-monitoring] adding namespace took 8.145552ms I0331 03:48:50.973955 4042 namespace.go:98] [openshift-apiserver] adding namespace I0331 03:48:50.974414 4042 namespace.go:102] [openshift-network-operator] adding namespace took 8.098543ms I0331 03:48:50.974442 4042 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0331 03:48:50.975090 4042 namespace.go:102] [openshift-backplane-cee] adding namespace took 8.32498ms I0331 03:48:50.975105 4042 namespace.go:98] [openshift-cluster-version] adding namespace I0331 03:48:50.975649 4042 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 8.287813ms I0331 03:48:50.975670 4042 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0331 03:48:50.976158 4042 namespace.go:102] [openshift-machine-config-operator] adding namespace took 8.220126ms I0331 03:48:50.976720 4042 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 8.332711ms I0331 03:48:50.976741 4042 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0331 03:48:50.977421 4042 namespace.go:102] [openshift-host-network] adding namespace took 8.508811ms I0331 03:48:50.977434 4042 namespace.go:98] [openshift-backplane] adding namespace I0331 03:48:50.978028 4042 namespace.go:102] [openshift-monitoring] adding namespace took 8.65077ms I0331 03:48:50.978043 4042 namespace.go:98] [openshift-dns-operator] adding namespace I0331 03:48:50.978622 4042 namespace.go:102] [openshift-operators-redhat] adding namespace took 8.737744ms I0331 03:48:50.978640 4042 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0331 03:48:50.979162 4042 namespace.go:102] [openshift-console] adding namespace took 8.693639ms I0331 03:48:50.979175 4042 namespace.go:98] [kube-node-lease] adding namespace I0331 03:48:50.979692 4042 namespace.go:102] [openshift-ingress-operator] adding namespace took 8.398978ms I0331 03:48:50.979708 4042 namespace.go:98] [openshift-kube-scheduler] adding namespace I0331 03:48:50.980316 4042 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 8.480258ms I0331 03:48:50.980334 4042 namespace.go:98] [openshift-controller-manager] adding namespace I0331 03:48:50.980828 4042 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.479341ms I0331 03:48:50.980847 4042 namespace.go:98] [openshift-infra] adding namespace I0331 03:48:50.981380 4042 namespace.go:102] [openshift-network-node-identity] adding namespace took 8.509324ms I0331 03:48:50.981402 4042 namespace.go:98] [openshift-must-gather-operator] adding namespace I0331 03:48:50.982032 4042 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 8.570866ms I0331 03:48:50.982048 4042 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0331 03:48:50.982489 4042 namespace.go:102] [openshift-apiserver] adding namespace took 8.527137ms I0331 03:48:50.982527 4042 namespace.go:98] [openshift-dns] adding namespace I0331 03:48:50.983024 4042 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 8.573497ms I0331 03:48:50.983042 4042 namespace.go:98] [openshift-logging] adding namespace I0331 03:48:50.983419 4042 namespace.go:102] [openshift-cluster-version] adding namespace took 8.308373ms I0331 03:48:50.983434 4042 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0331 03:48:50.984127 4042 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 8.450959ms I0331 03:48:50.984141 4042 namespace.go:98] [kube-public] adding namespace I0331 03:48:50.984749 4042 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 8.001758ms I0331 03:48:50.984768 4042 namespace.go:98] [openshift-operators] adding namespace I0331 03:48:50.985332 4042 namespace.go:102] [openshift-backplane] adding namespace took 7.890791ms I0331 03:48:50.985351 4042 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0331 03:48:50.985837 4042 namespace.go:102] [openshift-dns-operator] adding namespace took 7.786542ms I0331 03:48:50.985856 4042 namespace.go:98] [openshift-network-diagnostics] adding namespace I0331 03:48:50.986447 4042 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.796935ms I0331 03:48:50.986468 4042 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0331 03:48:50.986965 4042 namespace.go:102] [kube-node-lease] adding namespace took 7.784262ms I0331 03:48:50.986979 4042 namespace.go:98] [openshift-package-operator] adding namespace I0331 03:48:50.987466 4042 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.75201ms I0331 03:48:50.987488 4042 namespace.go:98] [openshift-marketplace] adding namespace I0331 03:48:50.988104 4042 namespace.go:102] [openshift-controller-manager] adding namespace took 7.763724ms I0331 03:48:50.988121 4042 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0331 03:48:50.988686 4042 namespace.go:102] [openshift-infra] adding namespace took 7.83217ms I0331 03:48:50.989208 4042 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.797668ms I0331 03:48:50.989227 4042 namespace.go:98] [openshift-node] adding namespace I0331 03:48:50.989812 4042 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.75806ms I0331 03:48:50.989829 4042 namespace.go:98] [openshift-ingress-canary] adding namespace I0331 03:48:50.990461 4042 namespace.go:102] [openshift-dns] adding namespace took 7.927303ms I0331 03:48:50.990947 4042 namespace.go:102] [openshift-logging] adding namespace took 7.897463ms I0331 03:48:50.990961 4042 namespace.go:98] [openshift-machine-api] adding namespace I0331 03:48:50.991404 4042 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.963415ms I0331 03:48:50.991421 4042 namespace.go:98] [default] adding namespace I0331 03:48:50.991989 4042 namespace.go:102] [kube-public] adding namespace took 7.840555ms I0331 03:48:50.992008 4042 namespace.go:98] [openshift-config-operator] adding namespace I0331 03:48:50.992437 4042 namespace.go:102] [openshift-operators] adding namespace took 7.663701ms I0331 03:48:50.992447 4042 namespace.go:98] [openshift-ovn-kubernetes] adding namespace W0331 03:48:50.992920 4042 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-k6mr9: pod openshift-network-diagnostics/network-check-target-k6mr9: no pod IPs found W0331 03:48:50.992934 4042 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-rr9tp: pod openshift-network-diagnostics/network-check-target-rr9tp: no pod IPs found W0331 03:48:50.992941 4042 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-znjtv: pod openshift-network-diagnostics/network-check-target-znjtv: no pod IPs found I0331 03:48:50.992972 4042 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.610057ms I0331 03:48:50.992988 4042 namespace.go:98] [openshift-service-ca-operator] adding namespace I0331 03:48:50.993520 4042 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.656974ms I0331 03:48:50.994098 4042 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.622579ms I0331 03:48:50.994526 4042 namespace.go:102] [openshift-package-operator] adding namespace took 7.539396ms I0331 03:48:50.995097 4042 namespace.go:102] [openshift-marketplace] adding namespace took 7.603086ms I0331 03:48:50.995112 4042 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0331 03:48:50.995609 4042 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.483755ms I0331 03:48:50.996078 4042 namespace.go:102] [openshift-node] adding namespace took 6.844989ms I0331 03:48:50.996093 4042 namespace.go:98] [openshift] adding namespace I0331 03:48:50.996587 4042 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.753122ms I0331 03:48:50.997104 4042 namespace.go:102] [openshift-machine-api] adding namespace took 6.136296ms I0331 03:48:50.997128 4042 namespace.go:98] [openshift-backplane-cse] adding namespace I0331 03:48:50.998042 4042 namespace.go:102] [default] adding namespace took 6.614306ms I0331 03:48:50.998061 4042 namespace.go:98] [openshift-config-managed] adding namespace I0331 03:48:50.998461 4042 namespace.go:102] [openshift-config-operator] adding namespace took 6.443642ms I0331 03:48:50.998918 4042 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.465594ms I0331 03:48:50.999303 4042 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.308675ms I0331 03:48:50.999787 4042 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 4.66811ms I0331 03:48:51.000381 4042 namespace.go:102] [openshift] adding namespace took 4.281105ms I0331 03:48:51.000805 4042 namespace.go:102] [openshift-backplane-cse] adding namespace took 3.644929ms I0331 03:48:51.001400 4042 namespace.go:102] [openshift-config-managed] adding namespace took 3.33231ms I0331 03:48:51.001858 4042 zone_ic_handler.go:186] Time taken to create transit switch: 326.756µs I0331 03:48:51.001941 4042 master.go:532] Adding or Updating local node "ip-10-0-1-243.ec2.internal" for network "default" I0331 03:48:51.002009 4042 obj_retry.go:544] Failed to create *v1.Node ip-10-0-2-160.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-160.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-160.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-160.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-160.ec2.internal I0331 03:48:51.002018 4042 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-143.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-143.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-143.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-143.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-143.ec2.internal I0331 03:48:51.005817 4042 default_network_controller.go:806] Node add failed for ip-10-0-1-243.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-243.ec2.internal, error getting gateway config for node ip-10-0-1-243.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-243.ec2.internal", failed to update chassis to local for local node ip-10-0-1-243.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-243.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-243.ec2.internal] E0331 03:48:51.005861 4042 obj_retry.go:541] Failed to create *v1.Node ip-10-0-1-243.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-243.ec2.internal, error getting gateway config for node ip-10-0-1-243.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-243.ec2.internal", failed to update chassis to local for local node ip-10-0-1-243.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-243.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-243.ec2.internal] I0331 03:48:51.005906 4042 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0331 03:48:51.005933 4042 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0331 03:48:51.005948 4042 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0331 03:48:51.005949 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-143.ec2.internal in network "default" I0331 03:48:51.005946 4042 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-1-243.ec2.internal", UID:"38898d8a-263f-48f7-807e-6fa1247ac502", APIVersion:"v1", ResourceVersion:"8588", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-243.ec2.internal, error getting gateway config for node ip-10-0-1-243.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-243.ec2.internal", failed to update chassis to local for local node ip-10-0-1-243.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-243.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-243.ec2.internal] I0331 03:48:51.005983 4042 node_tracker.go:235] Node ip-10-0-0-143.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-143.ec2.internal" I0331 03:48:51.006007 4042 node_tracker.go:169] Node ip-10-0-0-143.ec2.internal switch + router changed, syncing services in network "default" I0331 03:48:51.006021 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:48:51.006035 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-243.ec2.internal in network "default" I0331 03:48:51.006060 4042 node_tracker.go:235] Node ip-10-0-1-243.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-243.ec2.internal" I0331 03:48:51.006073 4042 node_tracker.go:169] Node ip-10-0-1-243.ec2.internal switch + router changed, syncing services in network "default" I0331 03:48:51.006080 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:48:51.006089 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-160.ec2.internal in network "default" I0331 03:48:51.006101 4042 node_tracker.go:235] Node ip-10-0-2-160.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-160.ec2.internal" I0331 03:48:51.006108 4042 node_tracker.go:169] Node ip-10-0-2-160.ec2.internal switch + router changed, syncing services in network "default" I0331 03:48:51.006115 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:48:51.006199 4042 iptables.go:108] Creating table: filter chain: FORWARD I0331 03:48:51.012577 4042 iptables.go:108] Creating table: filter chain: OUTPUT I0331 03:48:51.018819 4042 gateway_shared_intf.go:1787] Gateway Creation Complete I0331 03:48:51.018832 4042 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-243.ec2.internal","mac-address":"0a:ff:d3:0c:80:51","ip-addresses":["10.0.1.243/24"],"ip-address":"10.0.1.243/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:8ddab9f5-f11f-4a3d-a060-ce7862b7d841 k8s.ovn.org/node-encap-ips:["10.0.1.243"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.243/24"} k8s.ovn.org/zone-name:ip-10-0-1-243.ec2.internal] on node ip-10-0-1-243.ec2.internal I0331 03:48:51.034822 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-243.ec2.internal in network "default" I0331 03:48:51.034854 4042 master.go:532] Adding or Updating local node "ip-10-0-1-243.ec2.internal" for network "default" I0331 03:48:51.034876 4042 node_tracker.go:169] Node ip-10-0-1-243.ec2.internal switch + router changed, syncing services in network "default" I0331 03:48:51.034891 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:48:51.037924 4042 default_node_network_controller.go:518] Node connection status = connected W0331 03:48:51.039696 4042 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-wp5mx: pod kube-system/global-pull-secret-syncer-wp5mx: no pod IPs found W0331 03:48:51.039711 4042 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-9rdwh: pod openshift-multus/network-metrics-daemon-9rdwh: no pod IPs found W0331 03:48:51.039723 4042 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-k6mr9: pod openshift-network-diagnostics/network-check-target-k6mr9: no pod IPs found I0331 03:48:51.040813 4042 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-1-243.ec2.internal for the network default I0331 03:48:51.107007 4042 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0331 03:48:51.107034 4042 services_controller.go:208] Setting up event handlers for services for network=default I0331 03:48:51.107122 4042 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0331 03:48:51.107164 4042 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0331 03:48:51.107176 4042 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0331 03:48:51.207519 4042 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0331 03:48:51.207548 4042 repair.go:60] Starting repairing loop for services I0331 03:48:51.207649 4042 repair.go:131] Deleted 0 stale service LBs I0331 03:48:51.207669 4042 repair.go:137] Deleted 0 stale Chassis Template Vars I0331 03:48:51.207710 4042 repair.go:62] Finished repairing loop for services: 160.15µs I0331 03:48:51.207735 4042 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0331 03:48:51.207745 4042 services_controller.go:254] Starting workers for network=default I0331 03:48:51.208256 4042 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-znjtv, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-znjtv: suppressed error logged: pod openshift-network-diagnostics/network-check-target-znjtv: no pod IPs found I0331 03:48:51.208280 4042 obj_retry.go:544] Failed to create *v1.Pod kube-system/global-pull-secret-syncer-r9vx7, error: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-r9vx7: suppressed error logged: pod kube-system/global-pull-secret-syncer-r9vx7: no pod IPs found I0331 03:48:51.208268 4042 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-wp5mx] creating logical port kube-system_global-pull-secret-syncer-wp5mx for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:51.208326 4042 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-k6mr9] creating logical port openshift-network-diagnostics_network-check-target-k6mr9 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:51.208348 4042 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-lwlnf, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-lwlnf: suppressed error logged: pod openshift-multus/network-metrics-daemon-lwlnf: no pod IPs found I0331 03:48:51.208471 4042 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-9rdwh] creating logical port openshift-multus_network-metrics-daemon-9rdwh for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:51.208521 4042 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-46hqv, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-46hqv: suppressed error logged: pod openshift-multus/network-metrics-daemon-46hqv: no pod IPs found I0331 03:48:51.208526 4042 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-rr9tp, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-rr9tp: suppressed error logged: pod openshift-network-diagnostics/network-check-target-rr9tp: no pod IPs found I0331 03:48:51.208546 4042 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-k6mr9 I0331 03:48:51.208585 4042 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-9rdwh I0331 03:48:51.208585 4042 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-wp5mx I0331 03:48:51.226820 4042 pod.go:62] [openshift-network-diagnostics/network-check-target-k6mr9] pod update took 18.399214ms I0331 03:48:51.226840 4042 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-k6mr9] addLogicalPort annotation time took 18.424092ms I0331 03:48:51.227779 4042 pods.go:271] [openshift-network-diagnostics/network-check-target-k6mr9] addLogicalPort took 19.463835ms, libovsdb time 698.335µs I0331 03:48:51.233211 4042 pod.go:62] [kube-system/global-pull-secret-syncer-wp5mx] pod update took 24.776346ms I0331 03:48:51.233308 4042 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-wp5mx] addLogicalPort annotation time took 24.889007ms I0331 03:48:51.233312 4042 pod.go:62] [openshift-multus/network-metrics-daemon-9rdwh] pod update took 24.765558ms I0331 03:48:51.233327 4042 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-9rdwh] addLogicalPort annotation time took 24.780255ms I0331 03:48:51.234456 4042 pods.go:271] [kube-system/global-pull-secret-syncer-wp5mx] addLogicalPort took 26.205026ms, libovsdb time 867.144µs I0331 03:48:51.234859 4042 pods.go:271] [openshift-multus/network-metrics-daemon-9rdwh] addLogicalPort took 26.39814ms, libovsdb time 832.528µs I0331 03:48:51.235423 4042 admin_network_policy_controller.go:227] Starting controller default-network-controller I0331 03:48:51.235454 4042 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0331 03:48:51.235468 4042 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0331 03:48:51.235472 4042 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0331 03:48:51.235698 4042 repair.go:29] Repairing admin network policies took 221.264µs I0331 03:48:51.235827 4042 repair.go:92] Repairing baseline admin network policies took 118.297µs I0331 03:48:51.236220 4042 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0331 03:48:51.236251 4042 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0331 03:48:51.238941 4042 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0331 03:48:51.238966 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0331 03:48:51.238976 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0331 03:48:51.240634 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0331 03:48:51.240654 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0331 03:48:51.241103 4042 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0331 03:48:51.241129 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0331 03:48:51.241887 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0331 03:48:51.241917 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0331 03:48:51.241926 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0331 03:48:51.243295 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0331 03:48:51.243316 4042 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0331 03:48:51.243331 4042 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0331 03:48:51.244542 4042 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0331 03:48:51.244565 4042 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0331 03:48:51.244575 4042 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0331 03:48:51.244881 4042 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0331 03:48:51.244899 4042 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0331 03:48:51.244910 4042 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0331 03:48:51.246015 4042 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0331 03:48:51.246035 4042 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0331 03:48:51.246045 4042 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0331 03:48:51.248173 4042 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0331 03:48:51.248189 4042 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0331 03:48:51.248198 4042 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0331 03:48:51.248766 4042 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0331 03:48:51.248782 4042 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0331 03:48:51.248790 4042 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0331 03:48:51.251191 4042 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0331 03:48:51.251211 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0331 03:48:51.251220 4042 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0331 03:48:51.253034 4042 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0331 03:48:51.253061 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0331 03:48:51.253073 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0331 03:48:51.253621 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0331 03:48:51.253640 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0331 03:48:51.253647 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0331 03:48:51.254970 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0331 03:48:51.254989 4042 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0331 03:48:51.254998 4042 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0331 03:48:51.255525 4042 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0331 03:48:51.255542 4042 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0331 03:48:51.255552 4042 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0331 03:48:51.256541 4042 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0331 03:48:51.256559 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0331 03:48:51.256569 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0331 03:48:51.257078 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0331 03:48:51.257122 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0331 03:48:51.257132 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0331 03:48:51.257801 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0331 03:48:51.257817 4042 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0331 03:48:51.257825 4042 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0331 03:48:51.258690 4042 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0331 03:48:51.258707 4042 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0331 03:48:51.258716 4042 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0331 03:48:51.259733 4042 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0331 03:48:51.259756 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0331 03:48:51.259768 4042 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0331 03:48:51.260309 4042 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0331 03:48:51.260327 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0331 03:48:51.260335 4042 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0331 03:48:51.260838 4042 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0331 03:48:51.260855 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0331 03:48:51.260862 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0331 03:48:51.261396 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0331 03:48:51.261414 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0331 03:48:51.261422 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0331 03:48:51.261962 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0331 03:48:51.261978 4042 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0331 03:48:51.261987 4042 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0331 03:48:51.263660 4042 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0331 03:48:51.263678 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0331 03:48:51.263685 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0331 03:48:51.264185 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0331 03:48:51.264203 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0331 03:48:51.264622 4042 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0331 03:48:51.265352 4042 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0331 03:48:51.265373 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0331 03:48:51.265381 4042 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0331 03:48:51.265943 4042 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0331 03:48:51.265960 4042 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0331 03:48:51.265966 4042 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0331 03:48:51.266320 4042 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0331 03:48:51.266341 4042 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0331 03:48:51.266350 4042 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0331 03:48:51.266688 4042 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0331 03:48:51.272310 4042 egressfirewall.go:319] Starting EgressFirewall controller I0331 03:48:51.272340 4042 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0331 03:48:51.272359 4042 controller.go:133] Adding controller egress-firewall-controller event handlers I0331 03:48:51.272377 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0331 03:48:51.272390 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0331 03:48:51.272404 4042 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0331 03:48:51.272394 4042 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0331 03:48:51.272806 4042 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0331 03:48:51.272824 4042 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0331 03:48:51.272836 4042 egressqos.go:193] Setting up event handlers for EgressQoS I0331 03:48:51.272902 4042 egressfirewall.go:628] Syncing node "ip-10-0-1-243.ec2.internal" for egress firewall I0331 03:48:51.272913 4042 egressfirewall.go:628] Syncing node "ip-10-0-2-160.ec2.internal" for egress firewall I0331 03:48:51.272919 4042 egressfirewall.go:628] Syncing node "ip-10-0-0-143.ec2.internal" for egress firewall I0331 03:48:51.273047 4042 egressqos.go:245] Starting EgressQoS Controller I0331 03:48:51.273079 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0331 03:48:51.273090 4042 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0331 03:48:51.273095 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0331 03:48:51.273103 4042 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0331 03:48:51.273111 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0331 03:48:51.273126 4042 shared_informer.go:356] "Caches are synced" controller="egressqos" I0331 03:48:51.273133 4042 egressqos.go:259] Repairing EgressQoSes I0331 03:48:51.273138 4042 egressqos.go:399] Starting repairing loop for egressqos I0331 03:48:51.273225 4042 egressqos.go:401] Finished repairing loop for egressqos: 87.072µs I0331 03:48:51.273245 4042 egressservice_zone.go:132] Setting up event handlers for Egress Services I0331 03:48:51.273276 4042 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-243.ec2.internal I0331 03:48:51.273356 4042 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-243.ec2.internal : 82.609µs I0331 03:48:51.273604 4042 egressservice_zone.go:210] Starting Egress Services Controller I0331 03:48:51.273616 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0331 03:48:51.273623 4042 shared_informer.go:356] "Caches are synced" controller="egressservices" I0331 03:48:51.273628 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0331 03:48:51.273632 4042 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0331 03:48:51.273637 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0331 03:48:51.273642 4042 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0331 03:48:51.273648 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0331 03:48:51.273652 4042 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0331 03:48:51.273656 4042 egressservice_zone.go:228] Repairing Egress Services I0331 03:48:51.273915 4042 master_controller.go:88] Starting Admin Policy Based Route Controller I0331 03:48:51.273923 4042 external_controller.go:278] Starting Admin Policy Based Route Controller I0331 03:48:51.273998 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-160.ec2.internal I0331 03:48:51.274012 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-160.ec2.internal: 16.974µs I0331 03:48:51.274022 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-143.ec2.internal I0331 03:48:51.274028 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-143.ec2.internal: 5.967µs I0331 03:48:51.274034 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-243.ec2.internal I0331 03:48:51.274040 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-243.ec2.internal: 5.63µs I0331 03:48:51.274063 4042 default_network_controller.go:568] Completing all the Watchers took 317.08377ms I0331 03:48:51.274081 4042 default_network_controller.go:572] Starting unidling controllers I0331 03:48:51.274096 4042 unidle.go:46] Registering OVN SB ControllerEvent handler I0331 03:48:51.274108 4042 unidle.go:63] Populating Initial ContollerEvent events I0331 03:48:51.274120 4042 unidle.go:79] Setting up event handlers for services I0331 03:48:51.274196 4042 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0331 03:48:51.538471 4042 default_node_network_controller.go:518] Node connection status = connected I0331 03:48:52.038693 4042 default_node_network_controller.go:518] Node connection status = connected I0331 03:48:52.538724 4042 default_node_network_controller.go:518] Node connection status = connected I0331 03:48:53.038470 4042 default_node_network_controller.go:518] Node connection status = connected I0331 03:48:53.404816 4042 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0331 03:48:53.481906 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-160.ec2.internal I0331 03:48:53.481926 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-160.ec2.internal: 27.443µs I0331 03:48:53.481973 4042 egressfirewall.go:628] Syncing node "ip-10-0-2-160.ec2.internal" for egress firewall I0331 03:48:53.482032 4042 default_network_controller.go:1008] Node "ip-10-0-2-160.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:48:53.538377 4042 default_node_network_controller.go:518] Node connection status = connected I0331 03:48:53.553311 4042 bridgeconfig.go:537] Gateway is ready I0331 03:48:53.553328 4042 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0331 03:48:53.563103 4042 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.243. Updating OVN encap IP. I0331 03:48:53.567672 4042 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.243 - it is already configured I0331 03:48:53.567685 4042 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.243/24:{} 172.20.0.1/32:{}]. Updating annotations. I0331 03:48:53.567856 4042 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.243/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-243.ec2.internal","mac-address":"0a:ff:d3:0c:80:51","ip-addresses":["10.0.1.243/24"],"ip-address":"10.0.1.243/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:8ddab9f5-f11f-4a3d-a060-ce7862b7d841 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.243/24"}] on node ip-10-0-1-243.ec2.internal I0331 03:48:53.584046 4042 master.go:532] Adding or Updating local node "ip-10-0-1-243.ec2.internal" for network "default" I0331 03:48:53.584071 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-243.ec2.internal I0331 03:48:53.584083 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-243.ec2.internal: 16.87µs I0331 03:48:53.584085 4042 egressfirewall.go:628] Syncing node "ip-10-0-1-243.ec2.internal" for egress firewall I0331 03:48:53.584087 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-243.ec2.internal in network "default" I0331 03:48:53.584143 4042 node_tracker.go:169] Node ip-10-0-1-243.ec2.internal switch + router changed, syncing services in network "default" I0331 03:48:53.584159 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:48:53.584675 4042 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.1.243 I0331 03:48:53.584759 4042 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-243.ec2.internal I0331 03:48:53.588749 4042 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-243.ec2.internal. Updating no re-route policies I0331 03:48:53.588826 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-s22mb I0331 03:48:53.588835 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-243.ec2.internal I0331 03:48:53.588843 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-k6mr9 I0331 03:48:53.588833 4042 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-243.ec2.internal I0331 03:48:53.588855 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-k6mr9 I0331 03:48:53.588857 4042 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-kb8pp I0331 03:48:53.588857 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-9rdwh I0331 03:48:53.588862 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-zk2bx I0331 03:48:53.588875 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-zk2bx I0331 03:48:53.588876 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-9rdwh I0331 03:48:53.588876 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-46hqv I0331 03:48:53.588883 4042 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-k6mr9] creating logical port openshift-network-diagnostics_network-check-target-k6mr9 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:53.588884 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-znjtv I0331 03:48:53.588888 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-rr9tp I0331 03:48:53.588865 4042 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-kb8pp I0331 03:48:53.588844 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-lwlnf I0331 03:48:53.588894 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-24mct I0331 03:48:53.588875 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-srh5d I0331 03:48:53.588906 4042 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-kb8pp after 0 failed attempt(s) I0331 03:48:53.588908 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-srh5d I0331 03:48:53.588909 4042 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-48dxc I0331 03:48:53.588912 4042 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-9rdwh] creating logical port openshift-multus_network-metrics-daemon-9rdwh for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:53.588922 4042 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-48dxc I0331 03:48:53.588928 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-srh5d after 0 failed attempt(s) I0331 03:48:53.588900 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-rr9tp I0331 03:48:53.588943 4042 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-48dxc after 0 failed attempt(s) I0331 03:48:53.588847 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-s22mb I0331 03:48:53.588952 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-tx8lx I0331 03:48:53.588960 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-tx8lx I0331 03:48:53.588967 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-s22mb after 0 failed attempt(s) I0331 03:48:53.588877 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-9gr8g I0331 03:48:53.588977 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-ddxn8 I0331 03:48:53.588982 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-ddxn8 I0331 03:48:53.588892 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-znjtv I0331 03:48:53.589005 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-ddxn8 after 0 failed attempt(s) I0331 03:48:53.588971 4042 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-rr9tp, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-rr9tp: suppressed error logged: pod openshift-network-diagnostics/network-check-target-rr9tp: no pod IPs found I0331 03:48:53.589012 4042 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-znjtv, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-znjtv: suppressed error logged: pod openshift-network-diagnostics/network-check-target-znjtv: no pod IPs found I0331 03:48:53.588902 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-lwlnf I0331 03:48:53.589021 4042 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-r9vx7 I0331 03:48:53.589025 4042 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-r9vx7 I0331 03:48:53.588892 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-zk2bx after 0 failed attempt(s) I0331 03:48:53.589046 4042 obj_retry.go:375] Retry add failed for *v1.Pod kube-system/global-pull-secret-syncer-r9vx7, will try again later: failed to obtain IPs to add remote pod kube-system/global-pull-secret-syncer-r9vx7: suppressed error logged: pod kube-system/global-pull-secret-syncer-r9vx7: no pod IPs found I0331 03:48:53.589049 4042 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-wp5mx I0331 03:48:53.588971 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-tx8lx after 0 failed attempt(s) I0331 03:48:53.589056 4042 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-wp5mx I0331 03:48:53.588886 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-46hqv I0331 03:48:53.589088 4042 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-wp5mx] creating logical port kube-system_global-pull-secret-syncer-wp5mx for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:53.589092 4042 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-46hqv, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-46hqv: suppressed error logged: pod openshift-multus/network-metrics-daemon-46hqv: no pod IPs found I0331 03:48:53.588850 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-243.ec2.internal I0331 03:48:53.589125 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-243.ec2.internal after 0 failed attempt(s) I0331 03:48:53.589130 4042 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-c9tqk I0331 03:48:53.589145 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-c9tqk I0331 03:48:53.589158 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-c9tqk after 0 failed attempt(s) I0331 03:48:53.588983 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-9gr8g I0331 03:48:53.589175 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-9gr8g after 0 failed attempt(s) I0331 03:48:53.588866 4042 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-243.ec2.internal I0331 03:48:53.589193 4042 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-243.ec2.internal after 0 failed attempt(s) I0331 03:48:53.589039 4042 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-lwlnf, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-lwlnf: suppressed error logged: pod openshift-multus/network-metrics-daemon-lwlnf: no pod IPs found I0331 03:48:53.588912 4042 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-24mct I0331 03:48:53.589210 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-24mct after 0 failed attempt(s) I0331 03:48:53.589452 4042 pods.go:271] [openshift-network-diagnostics/network-check-target-k6mr9] addLogicalPort took 573.139µs, libovsdb time 232.099µs I0331 03:48:53.589465 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-k6mr9 after 0 failed attempt(s) I0331 03:48:53.589975 4042 pods.go:271] [openshift-multus/network-metrics-daemon-9rdwh] addLogicalPort took 1.073613ms, libovsdb time 281.325µs I0331 03:48:53.589986 4042 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-9rdwh after 0 failed attempt(s) I0331 03:48:53.590292 4042 pods.go:271] [kube-system/global-pull-secret-syncer-wp5mx] addLogicalPort took 1.215275ms, libovsdb time 197.544µs I0331 03:48:53.590311 4042 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-wp5mx after 0 failed attempt(s) I0331 03:48:53.599583 4042 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0331 03:48:53.610254 4042 iptables.go:108] Creating table: mangle chain: OUTPUT I0331 03:48:53.613246 4042 iptables.go:108] Creating table: nat chain: OUTPUT I0331 03:48:53.644460 4042 iptables.go:108] Creating table: nat chain: PREROUTING I0331 03:48:53.717526 4042 iptables.go:108] Creating table: filter chain: FORWARD I0331 03:48:53.733553 4042 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0331 03:48:53.733646 4042 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0331 03:48:53.733689 4042 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0331 03:48:53.733757 4042 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0331 03:48:53.733804 4042 gateway_init.go:325] Gateway and management port readiness took 2.699179638s I0331 03:48:53.733830 4042 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0331 03:48:53.733866 4042 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0331 03:48:53.733878 4042 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0331 03:48:53.733887 4042 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0331 03:48:53.733940 4042 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0331 03:48:53.734016 4042 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 50.715µs I0331 03:48:53.734227 4042 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0331 03:48:53.734251 4042 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.277µs I0331 03:48:53.734259 4042 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 33.61µs I0331 03:48:53.734265 4042 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0331 03:48:53.734274 4042 nad_controller.go:173] [node-nad-controller NAD controller]: started I0331 03:48:53.734279 4042 default_node_network_controller.go:898] Starting the default node network controller I0331 03:48:53.734284 4042 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0331 03:48:53.734308 4042 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 17.979µs I0331 03:48:53.734438 4042 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0331 03:48:53.734446 4042 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0331 03:48:53.734470 4042 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000aa7450 0xc0002dc230 0xc0002dc2a0 0xc0002dc380 0xc0002dc460 0xc0002dc4d0 0xc0002dc540 0xc000ab56d0 0xc0002dc5b0 0xc0002dc620 0xc0001a6410 0xc00550f260 0xc0002dc690 map[0x2f67040:0xc0023a5500 0x2f67580:0xc0023a5680 0x2f68000:0xc0023a5380 0x2f682a0:0xc0023a54a0 0x2f687e0:0xc0023a5560 0x2f68a80:0xc0023a55c0 0x2f68d20:0xc0023a5620 0x2f69500:0xc005506060 0x2f697a0:0xc0055060c0 0x2f93d40:0xc001505740 0x2f940e0:0xc0015057a0 0x2f94bc0:0xc000b8b980 0x2f96520:0xc001bda000 0x2f97000:0xc0015058c0 0x2f97e80:0xc001bda060] 0xc000aaa540 0} I0331 03:48:53.734542 4042 gateway.go:151] Starting gateway service sync I0331 03:48:53.734958 4042 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0331 03:48:53.737018 4042 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0331 03:48:53.750307 4042 egressfirewall.go:628] Syncing node "ip-10-0-0-143.ec2.internal" for egress firewall I0331 03:48:53.750346 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-143.ec2.internal I0331 03:48:53.750359 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-143.ec2.internal: 15.175µs I0331 03:48:53.750389 4042 default_network_controller.go:1008] Node "ip-10-0-0-143.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:48:53.758671 4042 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0331 03:48:53.759753 4042 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0331 03:48:53.760896 4042 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0331 03:48:53.794345 4042 gateway.go:168] Gateway service sync done. Time taken: 59.793129ms I0331 03:48:53.794754 4042 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31265 I0331 03:48:53.795158 4042 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000aa7450 0xc0002dc230 0xc0002dc2a0 0xc0002dc380 0xc0002dc460 0xc0002dc4d0 0xc0002dc540 0xc000ab56d0 0xc0002dc5b0 0xc0002dc620 0xc0001a6410 0xc00550f260 0xc0002dc690 map[0x2f67040:0xc0023a5500 0x2f67580:0xc0023a5680 0x2f68000:0xc0023a5380 0x2f682a0:0xc0023a54a0 0x2f687e0:0xc0023a5560 0x2f68a80:0xc0023a55c0 0x2f68d20:0xc0023a5620 0x2f69500:0xc005506060 0x2f697a0:0xc0055060c0 0x2f93d40:0xc001505740 0x2f940e0:0xc0015057a0 0x2f94bc0:0xc000b8b980 0x2f96520:0xc001bda000 0x2f97000:0xc0015058c0 0x2f97e80:0xc001bda060] 0xc000aaa540 0} I0331 03:48:53.795659 4042 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000aa7450 0xc0002dc230 0xc0002dc2a0 0xc0002dc380 0xc0002dc460 0xc0002dc4d0 0xc0002dc540 0xc000ab56d0 0xc0002dc5b0 0xc0002dc620 0xc0001a6410 0xc00550f260 0xc0002dc690 map[0x2f67040:0xc0023a5500 0x2f67580:0xc0023a5680 0x2f68000:0xc0023a5380 0x2f682a0:0xc0023a54a0 0x2f687e0:0xc0023a5560 0x2f68a80:0xc0023a55c0 0x2f68d20:0xc0023a5620 0x2f69500:0xc005506060 0x2f697a0:0xc0055060c0 0x2f93d40:0xc001505740 0x2f940e0:0xc0015057a0 0x2f94bc0:0xc000b8b980 0x2f96520:0xc001bda000 0x2f97000:0xc0015058c0 0x2f97e80:0xc001bda060] 0xc000aaa540 0} I0331 03:48:53.795691 4042 gateway.go:519] Reconciling gateway with updates I0331 03:48:53.800441 4042 gateway.go:326] Spawning Conntrack Rule Check Thread I0331 03:48:53.800462 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:48:53.800469 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:48:53.800474 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:48:53.800480 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:48:53.800485 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:48:53.800492 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:48:53.800516 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:48:53.800530 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:48:53.800537 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0331 03:48:53.800548 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0331 03:48:53.800553 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0331 03:48:53.800553 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:48:53.800557 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:48:53.800563 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:48:53.800566 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:48:53.800567 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:48:53.800572 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:48:53.800572 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:48:53.800574 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:48:53.800584 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:48:53.800586 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:48:53.800585 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0331 03:48:53.800600 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:48:53.800600 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:48:53.800604 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:48:53.800612 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:48:53.800558 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0331 03:48:53.800615 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:48:53.800643 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:48:53.800698 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:48:53.800649 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:48:53.800720 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:48:53.800729 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0331 03:48:53.800520 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:48:53.800747 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:48:53.800753 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0331 03:48:53.800719 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0331 03:48:53.800613 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:48:53.800539 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:48:53.800825 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0331 03:48:53.800556 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:48:53.800841 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:48:53.800573 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:48:53.800643 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0331 03:48:53.800870 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0331 03:48:53.800540 4042 default_node_network_controller.go:943] Gateway and management port readiness took 66.087407ms I0331 03:48:53.800884 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0331 03:48:53.800873 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0331 03:48:53.800571 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0331 03:48:53.800577 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:48:53.800996 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0331 03:48:53.800586 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0331 03:48:53.801010 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0331 03:48:53.801018 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0331 03:48:53.800589 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:48:53.801073 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:48:53.800615 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:48:53.800541 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:48:53.800623 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:48:53.801165 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:48:53.800626 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:48:53.801188 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:48:53.800632 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:48:53.801212 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:48:53.801217 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0331 03:48:53.800635 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0331 03:48:53.800636 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:48:53.801230 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:48:53.800636 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:48:53.801259 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:48:53.801263 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0331 03:48:53.800642 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0331 03:48:53.801271 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0331 03:48:53.800639 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:48:53.801277 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:48:53.801281 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0331 03:48:53.800639 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0331 03:48:53.801288 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0331 03:48:53.801299 4042 port_claim.go:71] Svc openshift-ingress/router-default: port 30196 is already open, no action needed I0331 03:48:53.801306 4042 port_claim.go:71] Svc openshift-ingress/router-default: port 30339 is already open, no action needed I0331 03:48:53.800541 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:48:53.801383 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:48:53.800645 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:48:53.801424 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:48:53.800650 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0331 03:48:53.800654 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:48:53.801467 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0331 03:48:53.800656 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:48:53.801474 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:48:53.800658 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0331 03:48:53.800657 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:48:53.801522 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:48:53.800656 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:48:53.801541 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:48:53.800656 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0331 03:48:53.801574 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0331 03:48:53.800661 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:48:53.801631 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:48:53.800666 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:48:53.801662 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:48:53.800669 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0331 03:48:53.801691 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0331 03:48:53.800671 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:48:53.801724 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:48:53.801741 4042 node_ip_handler_linux.go:160] Node IP manager is running I0331 03:48:53.800671 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0331 03:48:53.801783 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0331 03:48:53.800670 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:48:53.801809 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:48:53.800674 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:48:53.801839 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:48:53.801843 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0331 03:48:53.800660 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0331 03:48:53.800555 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:48:53.801851 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target W0331 03:48:53.836086 4042 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0331 03:48:53.836259 4042 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0331 03:48:53.836350 4042 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0331 03:48:53.843954 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0331 03:48:53.843986 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0331 03:48:53.844003 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0331 03:48:53.844020 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0331 03:48:53.844031 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0331 03:48:53.844037 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0331 03:48:53.844045 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0331 03:48:53.881355 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0331 03:48:53.881374 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0331 03:48:53.881381 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0331 03:48:53.881388 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0331 03:48:53.881394 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0331 03:48:53.881402 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0331 03:48:53.918205 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0331 03:48:53.918229 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0331 03:48:53.918236 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0331 03:48:53.918241 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0331 03:48:53.918248 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0331 03:48:53.918254 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0331 03:48:53.918260 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0331 03:48:53.918265 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0331 03:48:54.393954 4042 default_network_controller.go:1008] Node "ip-10-0-0-143.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:48:55.132051 4042 default_node_network_controller.go:1364] Starting node controller node sync W0331 03:48:55.132118 4042 egressip_healthcheck.go:76] Health checking using insecure connection I0331 03:48:55.132160 4042 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0331 03:48:55.143451 4042 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.380885ms I0331 03:48:55.143601 4042 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0331 03:48:55.143892 4042 egressservice_node.go:93] Setting up event handlers for Egress Services I0331 03:48:55.144072 4042 egressservice_node.go:183] Starting Egress Services Controller I0331 03:48:55.144088 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0331 03:48:55.144097 4042 shared_informer.go:356] "Caches are synced" controller="egressservices" I0331 03:48:55.144111 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0331 03:48:55.144118 4042 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0331 03:48:55.144124 4042 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0331 03:48:55.144128 4042 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0331 03:48:55.144132 4042 egressservice_node.go:197] Repairing Egress Services I0331 03:48:55.153806 4042 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0331 03:48:55.153817 4042 external_controller.go:278] Starting Admin Policy Based Route Controller I0331 03:48:55.154007 4042 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0331 03:48:55.154016 4042 link_network_manager.go:117] Link manager is running I0331 03:48:55.154024 4042 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0331 03:48:55.154147 4042 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0331 03:48:55.154382 4042 vrf_manager.go:128] VRF manager is running I0331 03:48:55.154456 4042 node_controller_manager.go:437] Removing flows to drop GARP I0331 03:48:55.154466 4042 gateway.go:519] Reconciling gateway with updates I0331 03:48:55.169409 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:48:55.169420 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:48:55.169430 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:48:55.169434 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:48:55.169434 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0331 03:48:55.169450 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0331 03:48:55.169450 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:48:55.169456 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:48:55.169466 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:48:55.169474 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:48:55.169479 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:48:55.169491 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:48:55.169509 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:48:55.169513 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:48:55.169514 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0331 03:48:55.169518 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:48:55.169526 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0331 03:48:55.169522 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:48:55.169536 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:48:55.169544 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:48:55.169550 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:48:55.169548 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:48:55.169554 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:48:55.169554 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:48:55.169559 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:48:55.169562 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:48:55.169570 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:48:55.169570 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:48:55.169418 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:48:55.169585 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0331 03:48:55.169585 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:48:55.169591 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0331 03:48:55.169555 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:48:55.169507 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:48:55.169597 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:48:55.169602 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:48:55.169603 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:48:55.169609 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:48:55.169610 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:48:55.169612 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:48:55.169610 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:48:55.169619 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:48:55.169622 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:48:55.169621 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:48:55.169623 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:48:55.169635 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:48:55.169638 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:48:55.169572 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:48:55.169646 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0331 03:48:55.169650 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:48:55.169655 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:48:55.169654 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:48:55.169655 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:48:55.169658 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0331 03:48:55.169667 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:48:55.169659 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0331 03:48:55.169536 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0331 03:48:55.169670 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0331 03:48:55.169546 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:48:55.169684 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:48:55.169700 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0331 03:48:55.169702 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0331 03:48:55.169586 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:48:55.169438 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:48:55.169715 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0331 03:48:55.169700 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0331 03:48:55.169716 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:48:55.169724 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:48:55.169624 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0331 03:48:55.169728 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0331 03:48:55.169563 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:48:55.169742 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0331 03:48:55.169755 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0331 03:48:55.169756 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0331 03:48:55.169636 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:48:55.169764 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0331 03:48:55.169528 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0331 03:48:55.169766 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:48:55.169545 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:48:55.169777 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:48:55.169780 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0331 03:48:55.169791 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0331 03:48:55.169792 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0331 03:48:55.169565 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:48:55.169799 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0331 03:48:55.169805 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:48:55.169577 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0331 03:48:55.169811 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0331 03:48:55.169559 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0331 03:48:55.169827 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0331 03:48:55.169843 4042 port_claim.go:71] Svc openshift-ingress/router-default: port 30196 is already open, no action needed I0331 03:48:55.169849 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0331 03:48:55.169860 4042 port_claim.go:71] Svc openshift-ingress/router-default: port 30339 is already open, no action needed I0331 03:48:55.169579 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:48:55.169870 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:48:55.169875 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0331 03:48:55.169592 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0331 03:48:55.169881 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0331 03:48:55.169830 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0331 03:48:55.169591 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:48:55.169911 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:48:55.169602 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0331 03:48:55.169940 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0331 03:48:55.169611 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:48:55.169971 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0331 03:48:55.169646 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:48:55.169645 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:48:55.169678 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0331 03:48:55.169681 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0331 03:48:55.169704 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0331 03:48:55.169717 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0331 03:48:55.169718 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0331 03:48:55.169721 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:48:55.169731 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:48:55.170057 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0331 03:48:55.169767 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0331 03:48:55.169528 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:48:55.170084 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0331 03:48:55.169545 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:48:55.170115 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:48:55.169577 4042 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:48:55.170177 4042 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:48:55.169480 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0331 03:48:55.192140 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0331 03:48:55.192148 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0331 03:48:55.192161 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0331 03:48:55.228965 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0331 03:48:55.228990 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0331 03:48:55.229010 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0331 03:48:55.229021 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0331 03:48:55.265213 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0331 03:48:55.265222 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0331 03:48:55.265238 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0331 03:48:55.265232 4042 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0331 03:48:57.433568 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-243.ec2.internal I0331 03:48:57.433594 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-243.ec2.internal: 34.681µs I0331 03:48:57.473141 4042 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-5946c4747f-vfbcf] creating logical port openshift-image-registry_image-registry-5946c4747f-vfbcf for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:57.473270 4042 kube.go:256] Updating pod openshift-image-registry/image-registry-5946c4747f-vfbcf I0331 03:48:57.493999 4042 pod.go:62] [openshift-image-registry/image-registry-5946c4747f-vfbcf] pod update took 20.756361ms I0331 03:48:57.494023 4042 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-5946c4747f-vfbcf] addLogicalPort annotation time took 20.786016ms I0331 03:48:57.495073 4042 pods.go:271] [openshift-image-registry/image-registry-5946c4747f-vfbcf] addLogicalPort took 21.946308ms, libovsdb time 808.318µs I0331 03:48:57.508780 4042 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-wfc49] creating logical port openshift-dns_dns-default-wfc49 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:57.508873 4042 kube.go:256] Updating pod openshift-dns/dns-default-wfc49 I0331 03:48:57.514215 4042 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-nvbbc] creating logical port openshift-ingress-canary_ingress-canary-nvbbc for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:57.514307 4042 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-nvbbc I0331 03:48:57.517814 4042 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-75sk7] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-75sk7 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:48:57.517965 4042 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-75sk7 I0331 03:48:57.526581 4042 pod.go:62] [openshift-dns/dns-default-wfc49] pod update took 17.725682ms I0331 03:48:57.526600 4042 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-wfc49] addLogicalPort annotation time took 17.745175ms I0331 03:48:57.527628 4042 pods.go:271] [openshift-dns/dns-default-wfc49] addLogicalPort took 18.858261ms, libovsdb time 776.965µs I0331 03:48:57.533330 4042 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75sk7] pod update took 15.386597ms I0331 03:48:57.533350 4042 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-75sk7] addLogicalPort annotation time took 15.408392ms I0331 03:48:57.533451 4042 pod.go:62] [openshift-ingress-canary/ingress-canary-nvbbc] pod update took 19.166233ms I0331 03:48:57.533469 4042 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-nvbbc] addLogicalPort annotation time took 19.184269ms I0331 03:48:57.534652 4042 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75sk7] addLogicalPort took 16.849883ms, libovsdb time 904.442µs I0331 03:48:57.535391 4042 pods.go:271] [openshift-ingress-canary/ingress-canary-nvbbc] addLogicalPort took 21.184858ms, libovsdb time 1.617473ms I0331 03:48:57.948225 4042 cni.go:403] [openshift-ingress-canary/ingress-canary-nvbbc 24303809b93c59423770dfca5881956882e806e77c7644c67ff3980808b0be83 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-nvbbc 24303809b93c59423770dfca5881956882e806e77c7644c67ff3980808b0be83 network default NAD default] I0331 03:48:57.979753 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-nvbbc, hostIfaceName: 24303809b93c594, network: default, NAD default, SandboxID: "24303809b93c59423770dfca5881956882e806e77c7644c67ff3980808b0be83", PCI device ID: , UID: "273a0947-a3f3-4591-8924-043259c010d5", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0331 03:48:57.980626 4042 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75sk7 bc1ceac4bb81366f30f0fcfae2d37392f8c0271b00d85fcf17f8b4058075708c network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75sk7 bc1ceac4bb81366f30f0fcfae2d37392f8c0271b00d85fcf17f8b4058075708c network default NAD default] I0331 03:48:57.988577 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-75sk7, hostIfaceName: bc1ceac4bb81366, network: default, NAD default, SandboxID: "bc1ceac4bb81366f30f0fcfae2d37392f8c0271b00d85fcf17f8b4058075708c", PCI device ID: , UID: "3f9c3510-5548-469e-9321-5aba442f09ba", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0331 03:48:58.048508 4042 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75sk7 bc1ceac4bb81366f30f0fcfae2d37392f8c0271b00d85fcf17f8b4058075708c network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-75sk7 bc1ceac4bb81366f30f0fcfae2d37392f8c0271b00d85fcf17f8b4058075708c network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:10:4a:4d:74:15\",\"name\":\"bc1ceac4bb81366\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/02b4f982-71e1-4a67-b847-248592203ed2\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:48:58.048591 4042 cni.go:424] [openshift-ingress-canary/ingress-canary-nvbbc 24303809b93c59423770dfca5881956882e806e77c7644c67ff3980808b0be83 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-nvbbc 24303809b93c59423770dfca5881956882e806e77c7644c67ff3980808b0be83 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:0a:01:2e:66:32\",\"name\":\"24303809b93c594\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/21620a99-11ac-4701-a8f5-c854bfc5c142\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:48:58.969995 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-143.ec2.internal in network "default" I0331 03:48:58.970016 4042 default_network_controller.go:1008] Node "ip-10-0-0-143.ec2.internal" in remote zone "ip-10-0-0-143.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:48:58.970031 4042 node_tracker.go:235] Node ip-10-0-0-143.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-143.ec2.internal" I0331 03:48:58.970047 4042 node_tracker.go:169] Node ip-10-0-0-143.ec2.internal switch + router changed, syncing services in network "default" I0331 03:48:58.970061 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:48:59.056917 4042 default_network_controller.go:1008] Node "ip-10-0-0-143.ec2.internal" in remote zone "ip-10-0-0-143.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:48:59.083375 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-143.ec2.internal in network "default" I0331 03:48:59.083419 4042 default_network_controller.go:1008] Node "ip-10-0-0-143.ec2.internal" in remote zone "ip-10-0-0-143.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:48:59.083434 4042 node_tracker.go:169] Node ip-10-0-0-143.ec2.internal switch + router changed, syncing services in network "default" I0331 03:48:59.083446 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:48:59.084068 4042 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-143.ec2.internal for the network default I0331 03:48:59.085275 4042 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-143.ec2.internal" on network "default" took: 1.21868ms I0331 03:48:59.417900 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-160.ec2.internal in network "default" I0331 03:48:59.417923 4042 node_tracker.go:235] Node ip-10-0-2-160.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-160.ec2.internal" I0331 03:48:59.417938 4042 node_tracker.go:169] Node ip-10-0-2-160.ec2.internal switch + router changed, syncing services in network "default" I0331 03:48:59.417952 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:48:59.418000 4042 default_network_controller.go:1008] Node "ip-10-0-2-160.ec2.internal" in remote zone "ip-10-0-2-160.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:48:59.492815 4042 default_network_controller.go:1008] Node "ip-10-0-2-160.ec2.internal" in remote zone "ip-10-0-2-160.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:48:59.531577 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-160.ec2.internal in network "default" I0331 03:48:59.531630 4042 node_tracker.go:169] Node ip-10-0-2-160.ec2.internal switch + router changed, syncing services in network "default" I0331 03:48:59.531642 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:48:59.531581 4042 default_network_controller.go:1008] Node "ip-10-0-2-160.ec2.internal" in remote zone "ip-10-0-2-160.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:48:59.532283 4042 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-160.ec2.internal for the network default I0331 03:48:59.533421 4042 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-160.ec2.internal" on network "default" took: 1.152789ms I0331 03:49:01.102693 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-160.ec2.internal I0331 03:49:01.102718 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-160.ec2.internal: 29.837µs I0331 03:49:01.102695 4042 egressfirewall.go:628] Syncing node "ip-10-0-2-160.ec2.internal" for egress firewall I0331 03:49:01.102691 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-160.ec2.internal in network "default" I0331 03:49:01.102745 4042 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-160.ec2.internal", IP: 10.0.2.160 to PMTUD blocking rules I0331 03:49:01.102759 4042 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-160.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:49:01.102798 4042 node_tracker.go:169] Node ip-10-0-2-160.ec2.internal switch + router changed, syncing services in network "default" I0331 03:49:01.102817 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:49:01.108955 4042 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-160.ec2.internal. Updating no re-route policies I0331 03:49:01.119228 4042 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-160.ec2.internal", IP: 10.0.2.160 to PMTUD blocking rules I0331 03:49:01.119247 4042 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-160.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:49:01.634557 4042 egressfirewall.go:628] Syncing node "ip-10-0-0-143.ec2.internal" for egress firewall I0331 03:49:01.634582 4042 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-143.ec2.internal in network "default" I0331 03:49:01.634640 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-143.ec2.internal I0331 03:49:01.634646 4042 node_tracker.go:169] Node ip-10-0-0-143.ec2.internal switch + router changed, syncing services in network "default" I0331 03:49:01.634653 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-143.ec2.internal: 17.01µs I0331 03:49:01.634667 4042 services_controller.go:574] Full service sync requested for network=default I0331 03:49:01.634703 4042 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-143.ec2.internal", IP: 10.0.0.143 to PMTUD blocking rules I0331 03:49:01.634718 4042 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-143.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:49:01.640711 4042 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-143.ec2.internal. Updating no re-route policies I0331 03:49:01.653862 4042 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-143.ec2.internal", IP: 10.0.0.143 to PMTUD blocking rules I0331 03:49:01.653876 4042 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-143.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:49:04.534450 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-143.ec2.internal I0331 03:49:04.534486 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-143.ec2.internal: 45.142µs I0331 03:49:05.383420 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-160.ec2.internal I0331 03:49:05.383454 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-160.ec2.internal: 41.43µs W0331 03:49:06.325259 4042 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:49:06.430782 4042 cni.go:403] [kube-system/global-pull-secret-syncer-wp5mx e24ed00a946d32d0b80d7a186cd355ce0948f981f21b211df130f07b5f4e04ea network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-wp5mx e24ed00a946d32d0b80d7a186cd355ce0948f981f21b211df130f07b5f4e04ea network default NAD default] I0331 03:49:06.439913 4042 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-wp5mx, hostIfaceName: e24ed00a946d32d, network: default, NAD default, SandboxID: "e24ed00a946d32d0b80d7a186cd355ce0948f981f21b211df130f07b5f4e04ea", PCI device ID: , UID: "8b2f516f-6ebc-47fe-a6c1-b59cbc7695b7", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0331 03:49:06.479237 4042 cni.go:424] [kube-system/global-pull-secret-syncer-wp5mx e24ed00a946d32d0b80d7a186cd355ce0948f981f21b211df130f07b5f4e04ea network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-wp5mx e24ed00a946d32d0b80d7a186cd355ce0948f981f21b211df130f07b5f4e04ea network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:cc:0b:fd:7f:ac\",\"name\":\"e24ed00a946d32d\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a99851cc-c4d0-46ab-ae13-0867d6931548\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:49:08.359237 4042 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-kb8pp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:49:08.359304 4042 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-kb8pp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:49:09.356212 4042 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-kb8pp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:49:09.356236 4042 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-kb8pp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0331 03:49:12.671196 4042 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:49:12.907990 4042 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:49:19.357588 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:49:26.253210 4042 cni.go:403] [openshift-network-diagnostics/network-check-target-k6mr9 4ea2642d08076f5c2eed6db9d5553d84b9cc0720fcaf7806eba4d4ab618d081c network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-k6mr9 4ea2642d08076f5c2eed6db9d5553d84b9cc0720fcaf7806eba4d4ab618d081c network default NAD default] I0331 03:49:26.261612 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-k6mr9, hostIfaceName: 4ea2642d08076f5, network: default, NAD default, SandboxID: "4ea2642d08076f5c2eed6db9d5553d84b9cc0720fcaf7806eba4d4ab618d081c", PCI device ID: , UID: "80e688fd-f652-459b-95d5-497766be2da0", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0331 03:49:26.301164 4042 cni.go:424] [openshift-network-diagnostics/network-check-target-k6mr9 4ea2642d08076f5c2eed6db9d5553d84b9cc0720fcaf7806eba4d4ab618d081c network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-k6mr9 4ea2642d08076f5c2eed6db9d5553d84b9cc0720fcaf7806eba4d4ab618d081c network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:e5:aa:cf:65:70\",\"name\":\"4ea2642d08076f5\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/36517a76-73eb-4e44-98c3-3f0b586e9bff\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:49:29.385482 4042 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:49:43.663694 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:49:43.663716 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:49:43.911612 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:49:43.911639 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0331 03:50:00.207321 4042 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29582150-b9wzk for pod on switch ip-10-0-1-243.ec2.internal I0331 03:50:00.207407 4042 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk I0331 03:50:00.225901 4042 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk] pod update took 18.516908ms I0331 03:50:00.225922 4042 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk] addLogicalPort annotation time took 18.542949ms I0331 03:50:00.227136 4042 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk] addLogicalPort took 19.832701ms, libovsdb time 929.826µs W0331 03:50:00.392063 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0331 03:50:00.581967 4042 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk 3ea181d2932d7199d149bd784fb9967b5ed96f15eee63049d1135395a7ac097c network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk 3ea181d2932d7199d149bd784fb9967b5ed96f15eee63049d1135395a7ac097c network default NAD default] I0331 03:50:00.591764 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29582150-b9wzk, hostIfaceName: 3ea181d2932d719, network: default, NAD default, SandboxID: "3ea181d2932d7199d149bd784fb9967b5ed96f15eee63049d1135395a7ac097c", PCI device ID: , UID: "79254e3e-fafe-4ecc-9f7d-de103c5c8ace", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0331 03:50:00.631513 4042 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk 3ea181d2932d7199d149bd784fb9967b5ed96f15eee63049d1135395a7ac097c network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk 3ea181d2932d7199d149bd784fb9967b5ed96f15eee63049d1135395a7ac097c network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:18:59:b1:2f:66\",\"name\":\"3ea181d2932d719\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/468865c2-1682-47a2-8c47-679f4f371ff6\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:01.849482 4042 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-6f994cf66-nxqw2] creating logical port openshift-ingress_router-default-6f994cf66-nxqw2 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:01.849643 4042 kube.go:256] Updating pod openshift-ingress/router-default-6f994cf66-nxqw2 I0331 03:51:01.867405 4042 pod.go:62] [openshift-ingress/router-default-6f994cf66-nxqw2] pod update took 17.833356ms I0331 03:51:01.867446 4042 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-6f994cf66-nxqw2] addLogicalPort annotation time took 17.862489ms I0331 03:51:01.868659 4042 pods.go:271] [openshift-ingress/router-default-6f994cf66-nxqw2] addLogicalPort took 19.191486ms, libovsdb time 908.537µs I0331 03:51:01.932321 4042 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-gfp9w] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-gfp9w for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:01.932406 4042 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-gfp9w I0331 03:51:01.956552 4042 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-gfp9w] pod update took 24.164076ms I0331 03:51:01.956576 4042 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-gfp9w] addLogicalPort annotation time took 24.19289ms I0331 03:51:01.957572 4042 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-gfp9w] addLogicalPort took 25.26801ms, libovsdb time 743.503µs I0331 03:51:05.304427 4042 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390vkrd8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:51:05.305011 4042 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390vkrd8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:51:05.305044 4042 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390vkrd8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 03:51:05.305057 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0331 03:51:05.305069 4042 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390vkrd8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 03:51:05.305075 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0331 03:51:06.195532 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0331 03:51:06.208598 4042 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390vkrd8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:51:06.208615 4042 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390vkrd8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:51:06.208636 4042 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390vkrd8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:51:06.208648 4042 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390vkrd8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:51:06.698643 4042 namespace.go:98] [openshift-service-ca] adding namespace I0331 03:51:06.699546 4042 namespace.go:102] [openshift-service-ca] adding namespace took 876.017µs I0331 03:51:06.735090 4042 namespace.go:142] [openshift-service-ca] updating namespace I0331 03:51:07.941170 4042 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0331 03:51:07.941987 4042 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 799.974µs I0331 03:51:07.971418 4042 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0331 03:51:09.308069 4042 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0331 03:51:09.326522 4042 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0331 03:51:13.616299 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0331 03:51:30.488601 4042 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:51:30.567161 4042 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-k84bg] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-k84bg for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:30.567271 4042 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-k84bg I0331 03:51:30.602887 4042 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-k84bg] pod update took 35.645768ms I0331 03:51:30.602915 4042 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-k84bg] addLogicalPort annotation time took 35.682368ms I0331 03:51:30.603962 4042 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-k84bg] addLogicalPort took 36.820117ms, libovsdb time 782.615µs I0331 03:51:30.611313 4042 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-l62f4] creating logical port openshift-insights_insights-runtime-extractor-l62f4 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:30.611389 4042 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-l62f4 I0331 03:51:30.654588 4042 pod.go:62] [openshift-insights/insights-runtime-extractor-l62f4] pod update took 43.224672ms I0331 03:51:30.654607 4042 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-l62f4] addLogicalPort annotation time took 43.247846ms I0331 03:51:30.655565 4042 pods.go:271] [openshift-insights/insights-runtime-extractor-l62f4] addLogicalPort took 44.262621ms, libovsdb time 715.32µs I0331 03:51:30.918051 4042 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-k84bg 8752e287dcc03859d35e69d156d6f62a35b90671cac39dd6f248792e32f7330d network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-k84bg 8752e287dcc03859d35e69d156d6f62a35b90671cac39dd6f248792e32f7330d network default NAD default] I0331 03:51:30.927078 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-k84bg, hostIfaceName: 8752e287dcc0385, network: default, NAD default, SandboxID: "8752e287dcc03859d35e69d156d6f62a35b90671cac39dd6f248792e32f7330d", PCI device ID: , UID: "bac106cf-5d7d-4d92-a0d8-eb450650c4ee", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0331 03:51:30.963012 4042 cni.go:403] [openshift-insights/insights-runtime-extractor-l62f4 c5067222911f8b47606d6a08116d29a4573bc63ffc93a9297086ac45c16d29fe network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-l62f4 c5067222911f8b47606d6a08116d29a4573bc63ffc93a9297086ac45c16d29fe network default NAD default] I0331 03:51:30.971142 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-l62f4, hostIfaceName: c5067222911f8b4, network: default, NAD default, SandboxID: "c5067222911f8b47606d6a08116d29a4573bc63ffc93a9297086ac45c16d29fe", PCI device ID: , UID: "4346c698-88db-4207-a56b-ba204df153f7", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0331 03:51:30.972265 4042 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-k84bg 8752e287dcc03859d35e69d156d6f62a35b90671cac39dd6f248792e32f7330d network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-k84bg 8752e287dcc03859d35e69d156d6f62a35b90671cac39dd6f248792e32f7330d network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:7d:e6:6a:2a:6a\",\"name\":\"8752e287dcc0385\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a413bc83-f304-4d93-8ae2-b6a9c39443df\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:31.011761 4042 cni.go:424] [openshift-insights/insights-runtime-extractor-l62f4 c5067222911f8b47606d6a08116d29a4573bc63ffc93a9297086ac45c16d29fe network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-l62f4 c5067222911f8b47606d6a08116d29a4573bc63ffc93a9297086ac45c16d29fe network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:22:43:42:8b:fc\",\"name\":\"c5067222911f8b4\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/05a3cde2-2968-44b2-afe2-37d3fea4f52c\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:31.668131 4042 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5946c4747f-vfbcf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:51:31.668173 4042 pods.go:176] Deleting pod: openshift-image-registry/image-registry-5946c4747f-vfbcf I0331 03:51:31.669615 4042 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-5946c4747f-vfbcf, ips: 10.128.0.6 I0331 03:51:31.669647 4042 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5946c4747f-vfbcf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:51:31.680117 4042 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7f9fbd477d-8bv9v] creating logical port openshift-image-registry_image-registry-7f9fbd477d-8bv9v for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:31.680279 4042 kube.go:256] Updating pod openshift-image-registry/image-registry-7f9fbd477d-8bv9v I0331 03:51:31.684836 4042 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5946c4747f-vfbcf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:51:31.684858 4042 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5946c4747f-vfbcf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:51:31.694838 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5946c4747f-vfbcf I0331 03:51:31.694868 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5946c4747f-vfbcf I0331 03:51:31.701089 4042 pod.go:62] [openshift-image-registry/image-registry-7f9fbd477d-8bv9v] pod update took 20.846111ms I0331 03:51:31.701115 4042 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7f9fbd477d-8bv9v] addLogicalPort annotation time took 20.875305ms I0331 03:51:31.702264 4042 pods.go:271] [openshift-image-registry/image-registry-7f9fbd477d-8bv9v] addLogicalPort took 22.15781ms, libovsdb time 863.31µs I0331 03:51:32.039736 4042 cni.go:403] [openshift-image-registry/image-registry-7f9fbd477d-8bv9v a6342bd5d5ae1aed860a883e6e471cde195d6a622a10fd66804555f3466fcc32 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-7f9fbd477d-8bv9v a6342bd5d5ae1aed860a883e6e471cde195d6a622a10fd66804555f3466fcc32 network default NAD default] I0331 03:51:32.048546 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7f9fbd477d-8bv9v, hostIfaceName: a6342bd5d5ae1ae, network: default, NAD default, SandboxID: "a6342bd5d5ae1aed860a883e6e471cde195d6a622a10fd66804555f3466fcc32", PCI device ID: , UID: "a3c55c1a-030d-4178-9661-48c97a194db2", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0331 03:51:32.089582 4042 cni.go:424] [openshift-image-registry/image-registry-7f9fbd477d-8bv9v a6342bd5d5ae1aed860a883e6e471cde195d6a622a10fd66804555f3466fcc32 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-7f9fbd477d-8bv9v a6342bd5d5ae1aed860a883e6e471cde195d6a622a10fd66804555f3466fcc32 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:b5:ad:b9:92:b9\",\"name\":\"a6342bd5d5ae1ae\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1557ebeb-a35f-4ad6-9ebd-fb087b2244f5\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:32.294432 4042 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7d65c97744-jwnj6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:51:32.295014 4042 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7d65c97744-jwnj6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:51:32.309378 4042 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7d65c97744-jwnj6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:51:32.309394 4042 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7d65c97744-jwnj6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:51:32.319848 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7d65c97744-jwnj6 I0331 03:51:32.319871 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7d65c97744-jwnj6 W0331 03:51:32.669689 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0331 03:51:32.698982 4042 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:51:32.723214 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0331 03:51:33.303581 4042 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:51:33.712884 4042 cni.go:403] [openshift-ingress/router-default-6f994cf66-nxqw2 cbad1c24371b0e0b83612515b7a4926caf5ddfd2bd568c881b4ccbad599d40d8 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-6f994cf66-nxqw2 cbad1c24371b0e0b83612515b7a4926caf5ddfd2bd568c881b4ccbad599d40d8 network default NAD default] I0331 03:51:33.721257 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-6f994cf66-nxqw2, hostIfaceName: cbad1c24371b0e0, network: default, NAD default, SandboxID: "cbad1c24371b0e0b83612515b7a4926caf5ddfd2bd568c881b4ccbad599d40d8", PCI device ID: , UID: "0ad1334c-cde1-4ac1-a9de-b8d8ebb67a64", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0331 03:51:33.760406 4042 cni.go:424] [openshift-ingress/router-default-6f994cf66-nxqw2 cbad1c24371b0e0b83612515b7a4926caf5ddfd2bd568c881b4ccbad599d40d8 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-6f994cf66-nxqw2 cbad1c24371b0e0b83612515b7a4926caf5ddfd2bd568c881b4ccbad599d40d8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:1b:8f:da:dd:bb\",\"name\":\"cbad1c24371b0e0\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ddfb26bb-b0ba-49ae-93ec-bc3f68a9f6f9\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:34.086488 4042 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-gfp9w 1a32ca648274311181d46a11251cd762f6b31c91c9610981c861bd331990adc1 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-gfp9w 1a32ca648274311181d46a11251cd762f6b31c91c9610981c861bd331990adc1 network default NAD default] I0331 03:51:34.095695 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-gfp9w, hostIfaceName: 1a32ca648274311, network: default, NAD default, SandboxID: "1a32ca648274311181d46a11251cd762f6b31c91c9610981c861bd331990adc1", PCI device ID: , UID: "f7494822-2937-4ed0-840c-29752ed8d657", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0331 03:51:34.136251 4042 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-gfp9w 1a32ca648274311181d46a11251cd762f6b31c91c9610981c861bd331990adc1 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-gfp9w 1a32ca648274311181d46a11251cd762f6b31c91c9610981c861bd331990adc1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:4c:d6:1e:73:63\",\"name\":\"1a32ca648274311\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/913678ee-4f19-4aa9-ae05-92ea3a4ba319\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:34.414941 4042 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-c56ff685-qhb5q] creating logical port openshift-monitoring_prometheus-operator-c56ff685-qhb5q for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:34.415034 4042 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-c56ff685-qhb5q I0331 03:51:34.433102 4042 pod.go:62] [openshift-monitoring/prometheus-operator-c56ff685-qhb5q] pod update took 18.094368ms I0331 03:51:34.433128 4042 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-c56ff685-qhb5q] addLogicalPort annotation time took 18.123929ms I0331 03:51:34.434412 4042 pods.go:271] [openshift-monitoring/prometheus-operator-c56ff685-qhb5q] addLogicalPort took 19.487589ms, libovsdb time 962.184µs W0331 03:51:34.686227 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0331 03:51:34.686246 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0331 03:51:34.686669 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:34.686686 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:34.686693 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:34.967857 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:51:34.967884 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:51:35.031542 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0331 03:51:35.031565 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0331 03:51:35.033293 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:35.033308 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:35.033314 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:35.064544 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:35.064613 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:35.064625 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:35.064713 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0331 03:51:35.064730 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0331 03:51:35.368520 4042 cni.go:403] [openshift-monitoring/prometheus-operator-c56ff685-qhb5q f0ecf206e790d3583a953c73da74a0446b27b158c86da136e2489fc5accf15e3 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-c56ff685-qhb5q f0ecf206e790d3583a953c73da74a0446b27b158c86da136e2489fc5accf15e3 network default NAD default] I0331 03:51:35.378612 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-c56ff685-qhb5q, hostIfaceName: f0ecf206e790d35, network: default, NAD default, SandboxID: "f0ecf206e790d3583a953c73da74a0446b27b158c86da136e2489fc5accf15e3", PCI device ID: , UID: "272c125e-45f7-471f-a0cc-fa04c8215bc9", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0331 03:51:35.418299 4042 cni.go:424] [openshift-monitoring/prometheus-operator-c56ff685-qhb5q f0ecf206e790d3583a953c73da74a0446b27b158c86da136e2489fc5accf15e3 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-c56ff685-qhb5q f0ecf206e790d3583a953c73da74a0446b27b158c86da136e2489fc5accf15e3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:15:ec:a4:e7:5e\",\"name\":\"f0ecf206e790d35\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a1e86636-2d90-4b88-ac60-a6945c27e39f\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:51:35.756866 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:35.756884 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:35.756889 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:51:35.757069 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0331 03:51:35.757081 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0331 03:51:35.863308 4042 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0331 03:51:40.003588 4042 base_network_controller_pods.go:487] [default/openshift-monitoring/openshift-state-metrics-bc865d6bb-kbx9j] creating logical port openshift-monitoring_openshift-state-metrics-bc865d6bb-kbx9j for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:40.003697 4042 kube.go:256] Updating pod openshift-monitoring/openshift-state-metrics-bc865d6bb-kbx9j I0331 03:51:40.022386 4042 pod.go:62] [openshift-monitoring/openshift-state-metrics-bc865d6bb-kbx9j] pod update took 18.711165ms I0331 03:51:40.022523 4042 base_network_controller_pods.go:937] [default/openshift-monitoring/openshift-state-metrics-bc865d6bb-kbx9j] addLogicalPort annotation time took 18.835002ms I0331 03:51:40.028856 4042 pods.go:271] [openshift-monitoring/openshift-state-metrics-bc865d6bb-kbx9j] addLogicalPort took 25.285907ms, libovsdb time 5.378609ms I0331 03:51:40.962689 4042 cni.go:403] [openshift-monitoring/openshift-state-metrics-bc865d6bb-kbx9j ff967d6cd8afb4169b1b29a0a394f2ca5fa6e28ec956e32ee87f8d7e5a294188 network default NAD default] ADD starting CNI request [openshift-monitoring/openshift-state-metrics-bc865d6bb-kbx9j ff967d6cd8afb4169b1b29a0a394f2ca5fa6e28ec956e32ee87f8d7e5a294188 network default NAD default] I0331 03:51:40.971556 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: openshift-state-metrics-bc865d6bb-kbx9j, hostIfaceName: ff967d6cd8afb41, network: default, NAD default, SandboxID: "ff967d6cd8afb4169b1b29a0a394f2ca5fa6e28ec956e32ee87f8d7e5a294188", PCI device ID: , UID: "f29c03e9-488b-4fba-b09e-d201c4e6f50d", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0331 03:51:41.013234 4042 cni.go:424] [openshift-monitoring/openshift-state-metrics-bc865d6bb-kbx9j ff967d6cd8afb4169b1b29a0a394f2ca5fa6e28ec956e32ee87f8d7e5a294188 network default NAD default] ADD finished CNI request [openshift-monitoring/openshift-state-metrics-bc865d6bb-kbx9j ff967d6cd8afb4169b1b29a0a394f2ca5fa6e28ec956e32ee87f8d7e5a294188 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:a3:23:10:f0:cd\",\"name\":\"ff967d6cd8afb41\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3b933d5d-4a53-4364-b274-816202c9564b\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:41.315410 4042 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:41.315579 4042 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0331 03:51:41.343068 4042 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 27.538072ms I0331 03:51:41.343088 4042 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 27.562539ms I0331 03:51:41.344118 4042 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 28.72757ms, libovsdb time 762.492µs I0331 03:51:41.678629 4042 cni.go:403] [openshift-monitoring/alertmanager-main-1 06335403174a080223599ff5e2bd2eac4ad4be4fbfbc3fb6241e23c0af4fec17 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 06335403174a080223599ff5e2bd2eac4ad4be4fbfbc3fb6241e23c0af4fec17 network default NAD default] I0331 03:51:41.689257 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 06335403174a080, network: default, NAD default, SandboxID: "06335403174a080223599ff5e2bd2eac4ad4be4fbfbc3fb6241e23c0af4fec17", PCI device ID: , UID: "50f4e059-9af9-4f17-b82b-b5ce25c21d03", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0331 03:51:41.734329 4042 cni.go:424] [openshift-monitoring/alertmanager-main-1 06335403174a080223599ff5e2bd2eac4ad4be4fbfbc3fb6241e23c0af4fec17 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 06335403174a080223599ff5e2bd2eac4ad4be4fbfbc3fb6241e23c0af4fec17 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:15:fe:90:bc:ff\",\"name\":\"06335403174a080\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d5df3aa4-858c-4747-a9c1-9d33ed0e3f13\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:43.219618 4042 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-5cbd5c4b4-tw2rp] creating logical port openshift-monitoring_thanos-querier-5cbd5c4b4-tw2rp for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:43.219722 4042 kube.go:256] Updating pod openshift-monitoring/thanos-querier-5cbd5c4b4-tw2rp I0331 03:51:43.254614 4042 pod.go:62] [openshift-monitoring/thanos-querier-5cbd5c4b4-tw2rp] pod update took 34.926338ms I0331 03:51:43.254637 4042 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-5cbd5c4b4-tw2rp] addLogicalPort annotation time took 34.95674ms I0331 03:51:43.255770 4042 pods.go:271] [openshift-monitoring/thanos-querier-5cbd5c4b4-tw2rp] addLogicalPort took 36.16579ms, libovsdb time 824.619µs I0331 03:51:43.579862 4042 cni.go:403] [openshift-monitoring/thanos-querier-5cbd5c4b4-tw2rp 050d7bd3d7460b02d9f37cb52d9aa8adc12ad082dbe7c746acfdf5a8a095341b network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-5cbd5c4b4-tw2rp 050d7bd3d7460b02d9f37cb52d9aa8adc12ad082dbe7c746acfdf5a8a095341b network default NAD default] I0331 03:51:43.588357 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-5cbd5c4b4-tw2rp, hostIfaceName: 050d7bd3d7460b0, network: default, NAD default, SandboxID: "050d7bd3d7460b02d9f37cb52d9aa8adc12ad082dbe7c746acfdf5a8a095341b", PCI device ID: , UID: "6f68756f-73c8-4efe-9dab-d815a1fdecb3", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0331 03:51:43.630258 4042 cni.go:424] [openshift-monitoring/thanos-querier-5cbd5c4b4-tw2rp 050d7bd3d7460b02d9f37cb52d9aa8adc12ad082dbe7c746acfdf5a8a095341b network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-5cbd5c4b4-tw2rp 050d7bd3d7460b02d9f37cb52d9aa8adc12ad082dbe7c746acfdf5a8a095341b network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:ea:f9:e5:c4:a2\",\"name\":\"050d7bd3d7460b0\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ffc9a5b2-aa41-4fe7-b4d5-41a429017f4b\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:51:43.783821 4042 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:51:43.783848 4042 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:51:43.783856 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0331 03:51:44.620041 4042 base_network_controller_pods.go:487] [default/openshift-console/console-6f96fcf57-7pmtm] creating logical port openshift-console_console-6f96fcf57-7pmtm for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:44.620132 4042 kube.go:256] Updating pod openshift-console/console-6f96fcf57-7pmtm I0331 03:51:44.660010 4042 pod.go:62] [openshift-console/console-6f96fcf57-7pmtm] pod update took 39.901734ms I0331 03:51:44.660029 4042 base_network_controller_pods.go:937] [default/openshift-console/console-6f96fcf57-7pmtm] addLogicalPort annotation time took 39.927192ms I0331 03:51:44.661312 4042 pods.go:271] [openshift-console/console-6f96fcf57-7pmtm] addLogicalPort took 41.286669ms, libovsdb time 932.518µs I0331 03:51:44.936163 4042 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-7f4548d649-22fvd] creating logical port openshift-monitoring_metrics-server-7f4548d649-22fvd for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:44.936294 4042 kube.go:256] Updating pod openshift-monitoring/metrics-server-7f4548d649-22fvd I0331 03:51:44.951390 4042 pod.go:62] [openshift-monitoring/metrics-server-7f4548d649-22fvd] pod update took 15.155588ms I0331 03:51:44.951411 4042 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-7f4548d649-22fvd] addLogicalPort annotation time took 15.183744ms I0331 03:51:44.952889 4042 pods.go:271] [openshift-monitoring/metrics-server-7f4548d649-22fvd] addLogicalPort took 16.740947ms, libovsdb time 1.070589ms I0331 03:51:44.984424 4042 cni.go:403] [openshift-console/console-6f96fcf57-7pmtm 32f2c99a581bb6bbaeefcf8be10081f117e2f357556e5ab09fef629e589ea01e network default NAD default] ADD starting CNI request [openshift-console/console-6f96fcf57-7pmtm 32f2c99a581bb6bbaeefcf8be10081f117e2f357556e5ab09fef629e589ea01e network default NAD default] I0331 03:51:44.993201 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6f96fcf57-7pmtm, hostIfaceName: 32f2c99a581bb6b, network: default, NAD default, SandboxID: "32f2c99a581bb6bbaeefcf8be10081f117e2f357556e5ab09fef629e589ea01e", PCI device ID: , UID: "659af41d-c6cd-45bd-b885-388d5db037ce", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0331 03:51:45.037303 4042 cni.go:424] [openshift-console/console-6f96fcf57-7pmtm 32f2c99a581bb6bbaeefcf8be10081f117e2f357556e5ab09fef629e589ea01e network default NAD default] ADD finished CNI request [openshift-console/console-6f96fcf57-7pmtm 32f2c99a581bb6bbaeefcf8be10081f117e2f357556e5ab09fef629e589ea01e network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:c1:9f:d8:67:b6\",\"name\":\"32f2c99a581bb6b\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dba833de-2c38-419f-a70c-03e41127178d\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:45.939256 4042 cni.go:403] [openshift-monitoring/metrics-server-7f4548d649-22fvd 23d6b41b36bc261cca4e8f3adc558f5471a3b662587bd667318a1bc86aee8d15 network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-7f4548d649-22fvd 23d6b41b36bc261cca4e8f3adc558f5471a3b662587bd667318a1bc86aee8d15 network default NAD default] I0331 03:51:45.947790 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-7f4548d649-22fvd, hostIfaceName: 23d6b41b36bc261, network: default, NAD default, SandboxID: "23d6b41b36bc261cca4e8f3adc558f5471a3b662587bd667318a1bc86aee8d15", PCI device ID: , UID: "f60b0713-ffe9-4717-bd93-c7cb804998d0", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0331 03:51:45.995419 4042 cni.go:424] [openshift-monitoring/metrics-server-7f4548d649-22fvd 23d6b41b36bc261cca4e8f3adc558f5471a3b662587bd667318a1bc86aee8d15 network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-7f4548d649-22fvd 23d6b41b36bc261cca4e8f3adc558f5471a3b662587bd667318a1bc86aee8d15 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:bc:b0:6c:d8:3a\",\"name\":\"23d6b41b36bc261\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/31e31ab4-5e6f-4c72-998a-5eb2684d09fb\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:51:46.493917 4042 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:51:46.494054 4042 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0331 03:51:46.529216 4042 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 35.220172ms I0331 03:51:46.529241 4042 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 35.250423ms I0331 03:51:46.530640 4042 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 36.744006ms, libovsdb time 994.037µs I0331 03:51:46.872952 4042 cni.go:403] [openshift-monitoring/prometheus-k8s-0 ac5e1eec3b8004ad31ae559ea878249f547e079f8f7c7fc6e34ccd88b6e6c9ab network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 ac5e1eec3b8004ad31ae559ea878249f547e079f8f7c7fc6e34ccd88b6e6c9ab network default NAD default] I0331 03:51:46.883868 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: ac5e1eec3b8004a, network: default, NAD default, SandboxID: "ac5e1eec3b8004ad31ae559ea878249f547e079f8f7c7fc6e34ccd88b6e6c9ab", PCI device ID: , UID: "989d58b7-ba63-4e35-9ebe-a2dc8f65dd2c", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0331 03:51:46.936552 4042 cni.go:424] [openshift-monitoring/prometheus-k8s-0 ac5e1eec3b8004ad31ae559ea878249f547e079f8f7c7fc6e34ccd88b6e6c9ab network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 ac5e1eec3b8004ad31ae559ea878249f547e079f8f7c7fc6e34ccd88b6e6c9ab network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:2a:82:91:34:6b\",\"name\":\"ac5e1eec3b8004a\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2513440f-f466-4cc7-9f5e-cf05d01e0875\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:51:47.778638 4042 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:47.778657 4042 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:47.778662 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:47.778666 4042 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:47.834757 4042 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:51:47.834784 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:51:48.096094 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0331 03:51:48.143954 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0331 03:51:48.369337 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0331 03:51:48.446116 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0331 03:51:48.784822 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0331 03:51:48.815184 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0331 03:51:49.111381 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:51:49.111405 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:51:50.401853 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0331 03:51:50.542742 4042 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:51:50.542762 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:51:51.126089 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0331 03:51:51.232375 4042 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:51:51.232391 4042 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:51:51.232397 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:51:51.454893 4042 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:51.454910 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:51.454915 4042 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:51.454919 4042 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0331 03:51:51.988760 4042 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0331 03:51:53.697982 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:51:53.819945 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:53.819968 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:53.819973 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:53.819978 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:51:55.816830 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:52:09.783902 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0331 03:52:11.739921 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:52:11.739944 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:52:11.739950 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0331 03:52:13.671319 4042 obj_retry.go:464] Detected object openshift-console/console-6cc7d4c5bc-dxmrx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:13.671914 4042 obj_retry.go:464] Detected object openshift-console/console-6cc7d4c5bc-dxmrx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:14.526278 4042 obj_retry.go:458] Detected object openshift-console/console-6cc7d4c5bc-dxmrx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:52:14.526302 4042 obj_retry.go:458] Detected object openshift-console/console-6cc7d4c5bc-dxmrx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:52:14.531443 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6cc7d4c5bc-dxmrx I0331 03:52:14.531462 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6cc7d4c5bc-dxmrx I0331 03:52:19.093172 4042 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7d65c97744-v2ct5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:19.093758 4042 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7d65c97744-v2ct5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:19.234590 4042 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7d65c97744-v2ct5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:52:19.234615 4042 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7d65c97744-v2ct5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:52:19.245144 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7d65c97744-v2ct5 I0331 03:52:19.245163 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7d65c97744-v2ct5 I0331 03:52:21.172848 4042 obj_retry.go:464] Detected object openshift-console/console-6cc7d4c5bc-srbsh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:21.173417 4042 obj_retry.go:464] Detected object openshift-console/console-6cc7d4c5bc-srbsh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:21.251892 4042 obj_retry.go:458] Detected object openshift-console/console-6cc7d4c5bc-srbsh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:52:21.251915 4042 obj_retry.go:458] Detected object openshift-console/console-6cc7d4c5bc-srbsh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:52:21.257815 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6cc7d4c5bc-srbsh I0331 03:52:21.257840 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6cc7d4c5bc-srbsh W0331 03:52:22.547126 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0331 03:52:25.271671 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0331 03:52:32.104522 4042 cni.go:403] [openshift-multus/network-metrics-daemon-9rdwh db6b410c56a3b4d8ab39b78bf0d7bf81754638549514e39777535457a53609f4 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-9rdwh db6b410c56a3b4d8ab39b78bf0d7bf81754638549514e39777535457a53609f4 network default NAD default] I0331 03:52:32.112807 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-9rdwh, hostIfaceName: db6b410c56a3b4d, network: default, NAD default, SandboxID: "db6b410c56a3b4d8ab39b78bf0d7bf81754638549514e39777535457a53609f4", PCI device ID: , UID: "e398b59c-e990-48a3-8e02-4c06428cad64", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0331 03:52:32.154512 4042 cni.go:424] [openshift-multus/network-metrics-daemon-9rdwh db6b410c56a3b4d8ab39b78bf0d7bf81754638549514e39777535457a53609f4 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-9rdwh db6b410c56a3b4d8ab39b78bf0d7bf81754638549514e39777535457a53609f4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:b8:8e:ee:99:d2\",\"name\":\"db6b410c56a3b4d\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/76d0f531-3be7-493f-be0c-5f55019608d3\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:52:37.600218 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0331 03:52:47.044674 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:52:47.044700 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0331 03:52:51.342723 4042 obj_retry.go:464] Detected object openshift-ingress/router-default-dd5548cd5-zpxzb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:51.343244 4042 obj_retry.go:464] Detected object openshift-ingress/router-default-dd5548cd5-zpxzb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:51.647207 4042 obj_retry.go:458] Detected object openshift-ingress/router-default-dd5548cd5-zpxzb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:52:51.647239 4042 obj_retry.go:458] Detected object openshift-ingress/router-default-dd5548cd5-zpxzb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:52:51.656356 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-dd5548cd5-zpxzb I0331 03:52:51.656378 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-dd5548cd5-zpxzb I0331 03:52:55.043412 4042 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk 3ea181d2932d7199d149bd784fb9967b5ed96f15eee63049d1135395a7ac097c network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk 3ea181d2932d7199d149bd784fb9967b5ed96f15eee63049d1135395a7ac097c network default NAD default] I0331 03:52:55.116430 4042 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk 3ea181d2932d7199d149bd784fb9967b5ed96f15eee63049d1135395a7ac097c network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk 3ea181d2932d7199d149bd784fb9967b5ed96f15eee63049d1135395a7ac097c network default NAD default], result "{}", err I0331 03:52:55.163136 4042 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:55.163170 4042 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk I0331 03:52:55.164397 4042 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk, ips: 10.128.0.10 I0331 03:52:55.164426 4042 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:52:56.208322 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:52:56.208355 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:01.818556 4042 cni.go:403] [openshift-monitoring/alertmanager-main-1 06335403174a080223599ff5e2bd2eac4ad4be4fbfbc3fb6241e23c0af4fec17 network default NAD default] DEL starting CNI request [openshift-monitoring/alertmanager-main-1 06335403174a080223599ff5e2bd2eac4ad4be4fbfbc3fb6241e23c0af4fec17 network default NAD default] I0331 03:53:01.897958 4042 cni.go:424] [openshift-monitoring/alertmanager-main-1 06335403174a080223599ff5e2bd2eac4ad4be4fbfbc3fb6241e23c0af4fec17 network default NAD default] DEL finished CNI request [openshift-monitoring/alertmanager-main-1 06335403174a080223599ff5e2bd2eac4ad4be4fbfbc3fb6241e23c0af4fec17 network default NAD default], result "{}", err I0331 03:53:01.962331 4042 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:01.962365 4042 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-1 I0331 03:53:01.963729 4042 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-1, ips: 10.128.0.18 I0331 03:53:01.963758 4042 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:02.077742 4042 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:02.077840 4042 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:02.087157 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0331 03:53:02.087188 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0331 03:53:02.142946 4042 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:53:02.143088 4042 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0331 03:53:02.172981 4042 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 29.932136ms I0331 03:53:02.173010 4042 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 29.96678ms I0331 03:53:02.174322 4042 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 31.391735ms, libovsdb time 953.682µs I0331 03:53:02.499490 4042 cni.go:403] [openshift-monitoring/alertmanager-main-1 88a6ac507670f14d36d387a3d75c6df7e48d0e743e00a010686ac7fc10013c0f network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 88a6ac507670f14d36d387a3d75c6df7e48d0e743e00a010686ac7fc10013c0f network default NAD default] I0331 03:53:02.507895 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 88a6ac507670f14, network: default, NAD default, SandboxID: "88a6ac507670f14d36d387a3d75c6df7e48d0e743e00a010686ac7fc10013c0f", PCI device ID: , UID: "6ef80de6-c072-4824-9176-98a5caaa1536", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0331 03:53:02.549625 4042 cni.go:424] [openshift-monitoring/alertmanager-main-1 88a6ac507670f14d36d387a3d75c6df7e48d0e743e00a010686ac7fc10013c0f network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 88a6ac507670f14d36d387a3d75c6df7e48d0e743e00a010686ac7fc10013c0f network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:1e:c2:06:42:1f\",\"name\":\"88a6ac507670f14\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6eca5f23-b37b-4666-8d91-ed2f4915a1b4\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:53:06.397362 4042 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:06.397981 4042 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:06.727943 4042 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:06.727967 4042 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:06.737107 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0331 03:53:06.737132 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0331 03:53:07.422075 4042 cni.go:403] [openshift-dns/dns-default-wfc49 666983ab3a01a182db54ea24be7b1f2a57f03742e8e543a96f8d32e992b67378 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-wfc49 666983ab3a01a182db54ea24be7b1f2a57f03742e8e543a96f8d32e992b67378 network default NAD default] I0331 03:53:07.430273 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-wfc49, hostIfaceName: 666983ab3a01a18, network: default, NAD default, SandboxID: "666983ab3a01a182db54ea24be7b1f2a57f03742e8e543a96f8d32e992b67378", PCI device ID: , UID: "ee166e8b-3c67-489d-a317-e40eb6267608", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0331 03:53:07.473182 4042 cni.go:424] [openshift-dns/dns-default-wfc49 666983ab3a01a182db54ea24be7b1f2a57f03742e8e543a96f8d32e992b67378 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-wfc49 666983ab3a01a182db54ea24be7b1f2a57f03742e8e543a96f8d32e992b67378 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:1f:36:14:21:68\",\"name\":\"666983ab3a01a18\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/115027cc-1731-4b31-8298-f535e0b7b3e0\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:53:20.131860 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0331 03:53:20.131893 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:53:20.131903 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:53:32.563810 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:53:32.563830 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:53:32.563837 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0331 03:53:33.858284 4042 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:33.858849 4042 obj_retry.go:464] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:34.010367 4042 obj_retry.go:464] Detected object openshift-console/console-6f96fcf57-t7nx4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:34.010909 4042 obj_retry.go:464] Detected object openshift-console/console-6f96fcf57-t7nx4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:34.525894 4042 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:34.525946 4042 obj_retry.go:458] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:34.534982 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0331 03:53:34.535008 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0331 03:53:34.867353 4042 obj_retry.go:458] Detected object openshift-console/console-6f96fcf57-t7nx4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:34.867373 4042 obj_retry.go:458] Detected object openshift-console/console-6f96fcf57-t7nx4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:34.874633 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6f96fcf57-t7nx4 I0331 03:53:34.874660 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6f96fcf57-t7nx4 I0331 03:53:44.675829 4042 cni.go:403] [openshift-console/console-6f96fcf57-7pmtm 32f2c99a581bb6bbaeefcf8be10081f117e2f357556e5ab09fef629e589ea01e network default NAD default] DEL starting CNI request [openshift-console/console-6f96fcf57-7pmtm 32f2c99a581bb6bbaeefcf8be10081f117e2f357556e5ab09fef629e589ea01e network default NAD default] I0331 03:53:44.743799 4042 cni.go:424] [openshift-console/console-6f96fcf57-7pmtm 32f2c99a581bb6bbaeefcf8be10081f117e2f357556e5ab09fef629e589ea01e network default NAD default] DEL finished CNI request [openshift-console/console-6f96fcf57-7pmtm 32f2c99a581bb6bbaeefcf8be10081f117e2f357556e5ab09fef629e589ea01e network default NAD default], result "{}", err I0331 03:53:44.797291 4042 obj_retry.go:464] Detected object openshift-console/console-6f96fcf57-7pmtm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:44.797321 4042 pods.go:176] Deleting pod: openshift-console/console-6f96fcf57-7pmtm I0331 03:53:44.798611 4042 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6f96fcf57-7pmtm, ips: 10.128.0.20 I0331 03:53:44.798642 4042 obj_retry.go:464] Detected object openshift-console/console-6f96fcf57-7pmtm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:53:45.256790 4042 obj_retry.go:458] Detected object openshift-console/console-6f96fcf57-7pmtm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:45.256816 4042 obj_retry.go:458] Detected object openshift-console/console-6f96fcf57-7pmtm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:53:45.265052 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6f96fcf57-7pmtm I0331 03:53:45.265083 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6f96fcf57-7pmtm I0331 03:54:09.188279 4042 cni.go:403] [openshift-monitoring/prometheus-k8s-0 ac5e1eec3b8004ad31ae559ea878249f547e079f8f7c7fc6e34ccd88b6e6c9ab network default NAD default] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 ac5e1eec3b8004ad31ae559ea878249f547e079f8f7c7fc6e34ccd88b6e6c9ab network default NAD default] I0331 03:54:09.257640 4042 cni.go:424] [openshift-monitoring/prometheus-k8s-0 ac5e1eec3b8004ad31ae559ea878249f547e079f8f7c7fc6e34ccd88b6e6c9ab network default NAD default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 ac5e1eec3b8004ad31ae559ea878249f547e079f8f7c7fc6e34ccd88b6e6c9ab network default NAD default], result "{}", err I0331 03:54:09.320344 4042 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:54:09.320375 4042 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0331 03:54:09.321567 4042 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.128.0.22 I0331 03:54:09.321593 4042 obj_retry.go:464] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:54:09.665323 4042 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:54:09.665346 4042 obj_retry.go:458] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:54:09.677978 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0331 03:54:09.678004 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0331 03:54:09.727566 4042 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-243.ec2.internal I0331 03:54:09.727734 4042 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0331 03:54:09.752781 4042 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 25.099464ms I0331 03:54:09.752803 4042 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 25.126691ms I0331 03:54:09.754086 4042 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 26.539065ms, libovsdb time 962.588µs I0331 03:54:10.087368 4042 cni.go:403] [openshift-monitoring/prometheus-k8s-0 21780b7c02ebb8f363cf6edae447d079356269f8c33eb55014adb5cfa597bf33 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 21780b7c02ebb8f363cf6edae447d079356269f8c33eb55014adb5cfa597bf33 network default NAD default] I0331 03:54:10.096474 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 21780b7c02ebb8f, network: default, NAD default, SandboxID: "21780b7c02ebb8f363cf6edae447d079356269f8c33eb55014adb5cfa597bf33", PCI device ID: , UID: "0410be33-9754-43c9-8fc8-b8c643369e94", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0331 03:54:10.138789 4042 cni.go:424] [openshift-monitoring/prometheus-k8s-0 21780b7c02ebb8f363cf6edae447d079356269f8c33eb55014adb5cfa597bf33 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 21780b7c02ebb8f363cf6edae447d079356269f8c33eb55014adb5cfa597bf33 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:88:c4:8e:9a:fa\",\"name\":\"21780b7c02ebb8f\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8e3e6a05-f88a-429a-9207-8a41dc1efa0e\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 03:54:27.124679 4042 egressfirewall.go:628] Syncing node "ip-10-0-0-143.ec2.internal" for egress firewall I0331 03:54:27.124771 4042 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-143.ec2.internal I0331 03:54:27.124801 4042 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-143.ec2.internal: 36.472µs I0331 03:54:35.623512 4042 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 W0331 03:55:10.664282 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:55:10.664305 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0331 03:55:18.814817 4042 base_network_controller_pods.go:487] [default/openshift-console/console-5cdf5d64f7-n7f4t] creating logical port openshift-console_console-5cdf5d64f7-n7f4t for pod on switch ip-10-0-1-243.ec2.internal I0331 03:55:18.814975 4042 kube.go:256] Updating pod openshift-console/console-5cdf5d64f7-n7f4t I0331 03:55:18.850998 4042 pod.go:62] [openshift-console/console-5cdf5d64f7-n7f4t] pod update took 36.070939ms I0331 03:55:18.851023 4042 base_network_controller_pods.go:937] [default/openshift-console/console-5cdf5d64f7-n7f4t] addLogicalPort annotation time took 36.101981ms I0331 03:55:18.852121 4042 pods.go:271] [openshift-console/console-5cdf5d64f7-n7f4t] addLogicalPort took 37.322123ms, libovsdb time 802.634µs I0331 03:55:19.171208 4042 cni.go:403] [openshift-console/console-5cdf5d64f7-n7f4t 44fe7d85db9997bdd2130fffad4e1cb8adce6eed14829cdffcd406fef4a9d14a network default NAD default] ADD starting CNI request [openshift-console/console-5cdf5d64f7-n7f4t 44fe7d85db9997bdd2130fffad4e1cb8adce6eed14829cdffcd406fef4a9d14a network default NAD default] I0331 03:55:19.180041 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5cdf5d64f7-n7f4t, hostIfaceName: 44fe7d85db9997b, network: default, NAD default, SandboxID: "44fe7d85db9997bdd2130fffad4e1cb8adce6eed14829cdffcd406fef4a9d14a", PCI device ID: , UID: "0fa57676-8976-4381-b3e3-f17641dd48ac", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0331 03:55:19.223762 4042 cni.go:424] [openshift-console/console-5cdf5d64f7-n7f4t 44fe7d85db9997bdd2130fffad4e1cb8adce6eed14829cdffcd406fef4a9d14a network default NAD default] ADD finished CNI request [openshift-console/console-5cdf5d64f7-n7f4t 44fe7d85db9997bdd2130fffad4e1cb8adce6eed14829cdffcd406fef4a9d14a network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:a3:20:1c:46:fc\",\"name\":\"44fe7d85db9997b\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6c439425-7276-4cb2-ae7d-cb4e2b9c5a17\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 03:55:29.696973 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0331 03:55:44.111206 4042 obj_retry.go:464] Detected object openshift-console/console-6b8cc4748d-79zsm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:55:44.111815 4042 obj_retry.go:464] Detected object openshift-console/console-6b8cc4748d-79zsm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:55:45.057793 4042 obj_retry.go:458] Detected object openshift-console/console-6b8cc4748d-79zsm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:55:45.057817 4042 obj_retry.go:458] Detected object openshift-console/console-6b8cc4748d-79zsm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:55:45.067207 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6b8cc4748d-79zsm I0331 03:55:45.067230 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6b8cc4748d-79zsm I0331 03:55:53.541810 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=640 I0331 03:55:53.628169 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0331 03:55:55.008239 4042 obj_retry.go:464] Detected object openshift-console/console-6b8cc4748d-qpt26 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:55:55.008854 4042 obj_retry.go:464] Detected object openshift-console/console-6b8cc4748d-qpt26 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:55:55.411060 4042 obj_retry.go:458] Detected object openshift-console/console-6b8cc4748d-qpt26 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:55:55.411085 4042 obj_retry.go:458] Detected object openshift-console/console-6b8cc4748d-qpt26 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:55:55.415584 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6b8cc4748d-qpt26 I0331 03:55:55.415605 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6b8cc4748d-qpt26 I0331 03:55:59.536349 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=76 I0331 03:56:05.534236 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0331 03:56:05.663274 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0331 03:56:11.537862 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=143 W0331 03:56:13.136744 4042 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:56:16.650666 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0331 03:56:24.201860 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0331 03:56:24.201941 4042 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-jdhq8 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-jdhq8: no pod IPs found I0331 03:56:36.645294 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0331 03:57:18.663317 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0331 03:57:28.534117 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=72 I0331 03:57:38.623067 4042 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0331 03:57:43.633081 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0331 03:58:16.656604 4042 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0331 03:58:17.537180 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=17 I0331 03:58:18.639309 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0331 04:00:00.204905 4042 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29582160-5kcsh for pod on switch ip-10-0-1-243.ec2.internal I0331 04:00:00.205012 4042 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh I0331 04:00:00.229690 4042 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh] pod update took 24.702398ms I0331 04:00:00.229711 4042 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh] addLogicalPort annotation time took 24.727063ms I0331 04:00:00.230877 4042 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh] addLogicalPort took 25.98936ms, libovsdb time 823.672µs I0331 04:00:00.580597 4042 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh 8569a72f0e695f728ab4e23d084e5b651a8de81886328636eec7212a9803d441 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh 8569a72f0e695f728ab4e23d084e5b651a8de81886328636eec7212a9803d441 network default NAD default] I0331 04:00:00.589022 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29582160-5kcsh, hostIfaceName: 8569a72f0e695f7, network: default, NAD default, SandboxID: "8569a72f0e695f728ab4e23d084e5b651a8de81886328636eec7212a9803d441", PCI device ID: , UID: "e4017a8a-8ccd-4105-9fcb-92be6bc0f440", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0331 04:00:00.631761 4042 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh 8569a72f0e695f728ab4e23d084e5b651a8de81886328636eec7212a9803d441 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh 8569a72f0e695f728ab4e23d084e5b651a8de81886328636eec7212a9803d441 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:03:38:a7:84:b3\",\"name\":\"8569a72f0e695f7\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/39c75a6f-4e70-499f-8152-6a881b2aa07e\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:00:03.786518 4042 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh 8569a72f0e695f728ab4e23d084e5b651a8de81886328636eec7212a9803d441 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh 8569a72f0e695f728ab4e23d084e5b651a8de81886328636eec7212a9803d441 network default NAD default] I0331 04:00:03.869343 4042 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh 8569a72f0e695f728ab4e23d084e5b651a8de81886328636eec7212a9803d441 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh 8569a72f0e695f728ab4e23d084e5b651a8de81886328636eec7212a9803d441 network default NAD default], result "{}", err I0331 04:00:03.914681 4042 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:03.914714 4042 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh I0331 04:00:03.915905 4042 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh, ips: 10.128.0.26 I0331 04:00:03.915937 4042 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:03.993214 4042 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29582160-t8rwp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:03.993597 4042 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29582160-t8rwp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.803816 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:04.803841 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-5kcsh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.007972 4042 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29582160-t8rwp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.008026 4042 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29582160-t8rwp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:23.625929 4042 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0331 04:01:26.744684 4042 namespace.go:98] [tsf] adding namespace I0331 04:01:26.745744 4042 namespace.go:102] [tsf] adding namespace took 1.030102ms I0331 04:01:26.763948 4042 namespace.go:142] [tsf] updating namespace I0331 04:01:26.779781 4042 namespace.go:142] [tsf] updating namespace I0331 04:01:30.706824 4042 namespace.go:98] [rhtpa-operator] adding namespace I0331 04:01:30.706895 4042 namespace.go:98] [konflux-operator] adding namespace I0331 04:01:30.707046 4042 namespace.go:98] [tsf-tpa] adding namespace I0331 04:01:30.707715 4042 namespace.go:102] [rhtpa-operator] adding namespace took 870.549µs I0331 04:01:30.708430 4042 namespace.go:102] [konflux-operator] adding namespace took 1.516006ms I0331 04:01:30.709075 4042 namespace.go:102] [tsf-tpa] adding namespace took 2.009154ms I0331 04:01:30.715173 4042 namespace.go:98] [cert-manager-operator] adding namespace I0331 04:01:30.715201 4042 namespace.go:98] [tsf-keycloak] adding namespace I0331 04:01:30.715331 4042 namespace.go:98] [konflux-ui] adding namespace I0331 04:01:30.715872 4042 namespace.go:102] [cert-manager-operator] adding namespace took 683.958µs I0331 04:01:30.716309 4042 namespace.go:102] [tsf-keycloak] adding namespace took 1.096551ms I0331 04:01:30.716797 4042 namespace.go:102] [konflux-ui] adding namespace took 1.447502ms I0331 04:01:30.727883 4042 namespace.go:98] [rhbk-operator] adding namespace I0331 04:01:30.727904 4042 namespace.go:98] [tsf-tas] adding namespace I0331 04:01:30.728455 4042 namespace.go:102] [rhbk-operator] adding namespace took 551.985µs I0331 04:01:30.728937 4042 namespace.go:102] [tsf-tas] adding namespace took 1.021808ms I0331 04:01:30.739732 4042 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:01:30.810131 4042 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:01:30.821479 4042 namespace.go:142] [konflux-operator] updating namespace I0331 04:01:30.836896 4042 namespace.go:142] [tsf-tpa] updating namespace I0331 04:01:30.848866 4042 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:01:30.857227 4042 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:30.921901 4042 namespace.go:142] [konflux-ui] updating namespace I0331 04:01:30.947403 4042 namespace.go:142] [rhbk-operator] updating namespace I0331 04:01:30.957806 4042 namespace.go:142] [tsf-tas] updating namespace I0331 04:01:30.969769 4042 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:01:31.047018 4042 namespace.go:142] [tsf-tpa] updating namespace I0331 04:01:31.086765 4042 namespace.go:142] [tsf-tpa] updating namespace I0331 04:01:31.109899 4042 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:01:31.132186 4042 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:01:31.160733 4042 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:31.181392 4042 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:31.209905 4042 namespace.go:142] [konflux-ui] updating namespace I0331 04:01:31.222072 4042 namespace.go:142] [konflux-ui] updating namespace I0331 04:01:31.237605 4042 namespace.go:142] [rhbk-operator] updating namespace I0331 04:01:31.260077 4042 namespace.go:142] [rhbk-operator] updating namespace I0331 04:01:31.273758 4042 namespace.go:142] [tsf-tas] updating namespace I0331 04:01:31.287264 4042 namespace.go:142] [tsf-tas] updating namespace I0331 04:01:31.304138 4042 namespace.go:142] [konflux-operator] updating namespace I0331 04:01:31.317924 4042 namespace.go:142] [konflux-operator] updating namespace I0331 04:01:32.047014 4042 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:01:32.068162 4042 namespace.go:142] [tsf-tpa] updating namespace I0331 04:01:38.908595 4042 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfv6dx9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:38.909153 4042 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfv6dx9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:38.909177 4042 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfv6dx9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:38.909184 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:38.909195 4042 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfv6dx9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:38.909200 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:39.928580 4042 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1k52vt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:39.929136 4042 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1k52vt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:39.929162 4042 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1k52vt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:39.929169 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:39.929183 4042 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1k52vt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:39.929187 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:39.933858 4042 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfv6dx9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:39.933878 4042 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfv6dx9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:39.933888 4042 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfv6dx9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:39.933901 4042 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bfv6dx9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:40.959798 4042 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1k52vt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:40.959828 4042 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1k52vt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:40.959849 4042 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1k52vt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:40.959863 4042 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1k52vt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:43.973608 4042 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mbbng of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:43.974092 4042 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mbbng of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:43.974117 4042 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mbbng of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:43.974125 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:43.974141 4042 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mbbng of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:43.974145 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:44.011173 4042 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sx2mw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:44.011576 4042 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sx2mw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:44.011595 4042 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sx2mw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:44.011602 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:44.011613 4042 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sx2mw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:44.011617 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:44.999226 4042 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf527tz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:44.999779 4042 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf527tz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:44.999807 4042 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf527tz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:44.999815 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:44.999827 4042 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf527tz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:44.999831 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:45.020201 4042 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mbbng of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:45.020222 4042 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mbbng of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:45.020236 4042 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mbbng of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:45.020254 4042 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mbbng of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:45.035573 4042 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7tp6wc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:45.036053 4042 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7tp6wc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:45.036075 4042 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7tp6wc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:45.036084 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:01:45.036100 4042 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7tp6wc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:01:45.036106 4042 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:01:45.066273 4042 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sx2mw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:45.066292 4042 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sx2mw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:45.066304 4042 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sx2mw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:45.066315 4042 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8sx2mw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:46.026041 4042 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf527tz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:46.026066 4042 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf527tz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:46.026087 4042 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf527tz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:46.026101 4042 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf527tz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:46.068019 4042 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7tp6wc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:46.068042 4042 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7tp6wc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:46.068058 4042 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7tp6wc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:46.068071 4042 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7tp6wc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:47.536985 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0331 04:01:53.626357 4042 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gf9fp] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-gf9fp for pod on switch ip-10-0-1-243.ec2.internal I0331 04:01:53.626488 4042 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gf9fp I0331 04:01:53.662677 4042 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gf9fp] pod update took 36.218285ms I0331 04:01:53.662701 4042 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gf9fp] addLogicalPort annotation time took 36.246686ms I0331 04:01:53.663845 4042 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gf9fp] addLogicalPort took 37.50251ms, libovsdb time 838.612µs I0331 04:01:53.726581 4042 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-nk4zr for pod on switch ip-10-0-1-243.ec2.internal I0331 04:01:53.726696 4042 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr I0331 04:01:53.749260 4042 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr] pod update took 22.585323ms I0331 04:01:53.749280 4042 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr] addLogicalPort annotation time took 22.611345ms I0331 04:01:53.750401 4042 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr] addLogicalPort took 23.837699ms, libovsdb time 775.081µs I0331 04:01:53.982273 4042 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gf9fp b83209e49ed705acb8e22ca87590a0b6aa3578a536f98b5349c41ef41e219e5a network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gf9fp b83209e49ed705acb8e22ca87590a0b6aa3578a536f98b5349c41ef41e219e5a network default NAD default] I0331 04:01:53.991053 4042 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-gf9fp, hostIfaceName: b83209e49ed705a, network: default, NAD default, SandboxID: "b83209e49ed705acb8e22ca87590a0b6aa3578a536f98b5349c41ef41e219e5a", PCI device ID: , UID: "a1b1486d-ade8-40c8-badf-a24a7201dbb6", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0331 04:01:54.034210 4042 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gf9fp b83209e49ed705acb8e22ca87590a0b6aa3578a536f98b5349c41ef41e219e5a network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-gf9fp b83209e49ed705acb8e22ca87590a0b6aa3578a536f98b5349c41ef41e219e5a network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:a9:70:5f:5e:d2\",\"name\":\"b83209e49ed705a\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/265639bf-5b15-4726-b3ad-081ffa7d8944\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:01:54.090488 4042 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr cf1f13ed26575336c254197a1b7d9a977a035625a5d2cccff4241148b9df852d network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr cf1f13ed26575336c254197a1b7d9a977a035625a5d2cccff4241148b9df852d network default NAD default] I0331 04:01:54.098625 4042 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-nk4zr, hostIfaceName: cf1f13ed2657533, network: default, NAD default, SandboxID: "cf1f13ed26575336c254197a1b7d9a977a035625a5d2cccff4241148b9df852d", PCI device ID: , UID: "c4455ff3-3b1a-4114-9f6c-e71d07a8fcfa", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0331 04:01:54.140856 4042 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr cf1f13ed26575336c254197a1b7d9a977a035625a5d2cccff4241148b9df852d network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr cf1f13ed26575336c254197a1b7d9a977a035625a5d2cccff4241148b9df852d network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:95:36:d7:d0:a6\",\"name\":\"cf1f13ed2657533\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/487daed2-05bb-49f7-8ade-08d1eb2741d4\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:01:58.337898 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0331 04:01:58.869868 4042 namespace.go:98] [cert-manager] adding namespace I0331 04:01:58.870750 4042 namespace.go:102] [cert-manager] adding namespace took 860.165µs I0331 04:01:58.904729 4042 namespace.go:142] [cert-manager] updating namespace I0331 04:01:58.919804 4042 namespace.go:142] [cert-manager] updating namespace I0331 04:02:00.691757 4042 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-5m2v7] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-5m2v7 for pod on switch ip-10-0-1-243.ec2.internal I0331 04:02:00.691866 4042 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-5m2v7 I0331 04:02:00.706316 4042 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-5m2v7] pod update took 14.476421ms I0331 04:02:00.706338 4042 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-5m2v7] addLogicalPort annotation time took 14.501953ms I0331 04:02:00.707440 4042 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-5m2v7] addLogicalPort took 15.699412ms, libovsdb time 776.749µs I0331 04:02:01.063216 4042 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-5m2v7 7012eca482ffc0eeaea916a03a182192de0cbfdca4e225dd0a0c1f2a446da273 network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-5m2v7 7012eca482ffc0eeaea916a03a182192de0cbfdca4e225dd0a0c1f2a446da273 network default NAD default] I0331 04:02:01.071673 4042 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-5m2v7, hostIfaceName: 7012eca482ffc0e, network: default, NAD default, SandboxID: "7012eca482ffc0eeaea916a03a182192de0cbfdca4e225dd0a0c1f2a446da273", PCI device ID: , UID: "663187c2-7081-4ad5-a00d-82d8f8f0b568", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0331 04:02:01.114115 4042 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-5m2v7 7012eca482ffc0eeaea916a03a182192de0cbfdca4e225dd0a0c1f2a446da273 network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-5m2v7 7012eca482ffc0eeaea916a03a182192de0cbfdca4e225dd0a0c1f2a446da273 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:05:13:89:d8:69\",\"name\":\"7012eca482ffc0e\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/011e4518-8602-40d2-b38c-9cfb90b539cb\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:02:02.803528 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:02:02.830872 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:02:03.011340 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:02:03.025971 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:02:03.072726 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0331 04:02:03.072748 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0331 04:02:04.295722 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0331 04:02:06.311084 4042 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5b4bf68748-2hx4j] creating logical port cert-manager_cert-manager-cainjector-5b4bf68748-2hx4j for pod on switch ip-10-0-1-243.ec2.internal I0331 04:02:06.311182 4042 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5b4bf68748-2hx4j I0331 04:02:06.326616 4042 pod.go:62] [cert-manager/cert-manager-cainjector-5b4bf68748-2hx4j] pod update took 15.461857ms I0331 04:02:06.326638 4042 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-cainjector-5b4bf68748-2hx4j] addLogicalPort annotation time took 15.489699ms I0331 04:02:06.327775 4042 pods.go:271] [cert-manager/cert-manager-cainjector-5b4bf68748-2hx4j] addLogicalPort took 16.708053ms, libovsdb time 817.99µs I0331 04:02:06.686113 4042 cni.go:403] [cert-manager/cert-manager-cainjector-5b4bf68748-2hx4j a9c22b79b26b84b200baae31c6dd5ea5e141f2758664e990428404b48445e577 network default NAD default] ADD starting CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-2hx4j a9c22b79b26b84b200baae31c6dd5ea5e141f2758664e990428404b48445e577 network default NAD default] I0331 04:02:06.694521 4042 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5b4bf68748-2hx4j, hostIfaceName: a9c22b79b26b84b, network: default, NAD default, SandboxID: "a9c22b79b26b84b200baae31c6dd5ea5e141f2758664e990428404b48445e577", PCI device ID: , UID: "ef0a19ac-292d-486f-ada2-5616060a5a9d", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0331 04:02:06.738072 4042 cni.go:424] [cert-manager/cert-manager-cainjector-5b4bf68748-2hx4j a9c22b79b26b84b200baae31c6dd5ea5e141f2758664e990428404b48445e577 network default NAD default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-2hx4j a9c22b79b26b84b200baae31c6dd5ea5e141f2758664e990428404b48445e577 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:aa:d2:d1:d0:39\",\"name\":\"a9c22b79b26b84b\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4fd2fe8-cb85-4116-8e23-db875ce07ad0\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:07.117600 4042 namespace.go:142] [openshift-operators] updating namespace I0331 04:02:07.132336 4042 namespace.go:142] [openshift-operators] updating namespace W0331 04:02:09.286857 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0331 04:02:13.107843 4042 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8m57x] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-8m57x for pod on switch ip-10-0-1-243.ec2.internal I0331 04:02:13.107977 4042 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8m57x I0331 04:02:13.133180 4042 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8m57x] pod update took 25.229073ms I0331 04:02:13.133200 4042 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8m57x] addLogicalPort annotation time took 25.252469ms I0331 04:02:13.134374 4042 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8m57x] addLogicalPort took 26.547554ms, libovsdb time 838.352µs I0331 04:02:13.770932 4042 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8m57x a2ef0398e7823e3c8e9bd5b0d6c827411d9b34ff7faf5cc79154358ea2de6152 network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8m57x a2ef0398e7823e3c8e9bd5b0d6c827411d9b34ff7faf5cc79154358ea2de6152 network default NAD default] I0331 04:02:13.779507 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-8m57x, hostIfaceName: a2ef0398e7823e3, network: default, NAD default, SandboxID: "a2ef0398e7823e3c8e9bd5b0d6c827411d9b34ff7faf5cc79154358ea2de6152", PCI device ID: , UID: "16494ef5-93d3-4d8f-a8a2-431cdbf293c8", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0331 04:02:13.822641 4042 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8m57x a2ef0398e7823e3c8e9bd5b0d6c827411d9b34ff7faf5cc79154358ea2de6152 network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-8m57x a2ef0398e7823e3c8e9bd5b0d6c827411d9b34ff7faf5cc79154358ea2de6152 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:c9:5f:28:24:20\",\"name\":\"a2ef0398e7823e3\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/588a0f85-4b7d-45e2-8f32-80ebf6ec6f67\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:02:15.306106 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0331 04:02:16.214329 4042 namespace.go:98] [trusted-artifact-signer] adding namespace I0331 04:02:16.215121 4042 namespace.go:102] [trusted-artifact-signer] adding namespace took 768.27µs I0331 04:02:16.233341 4042 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:02:16.245178 4042 namespace.go:142] [trusted-artifact-signer] updating namespace W0331 04:02:16.416738 4042 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0331 04:02:17.982338 4042 namespace.go:142] [openshift-operators] updating namespace I0331 04:02:18.445087 4042 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-57qr6] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-57qr6 for pod on switch ip-10-0-1-243.ec2.internal I0331 04:02:18.445191 4042 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-57qr6 I0331 04:02:18.464148 4042 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-57qr6] pod update took 18.978257ms I0331 04:02:18.464172 4042 base_network_controller_pods.go:937] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-57qr6] addLogicalPort annotation time took 19.004077ms I0331 04:02:18.465380 4042 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-57qr6] addLogicalPort took 20.306243ms, libovsdb time 835.019µs I0331 04:02:18.809883 4042 cni.go:403] [openshift-operators/tekton-operator-webhook-668b6fdd5d-57qr6 4c2979e1b01aed7a7084fbbf983cf7a307492c7954672d0cb3c6b0d077a64108 network default NAD default] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-57qr6 4c2979e1b01aed7a7084fbbf983cf7a307492c7954672d0cb3c6b0d077a64108 network default NAD default] I0331 04:02:18.818857 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-57qr6, hostIfaceName: 4c2979e1b01aed7, network: default, NAD default, SandboxID: "4c2979e1b01aed7a7084fbbf983cf7a307492c7954672d0cb3c6b0d077a64108", PCI device ID: , UID: "52b4cc8d-5f4f-4afe-afa7-1dcc5fad2843", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0331 04:02:18.862511 4042 cni.go:424] [openshift-operators/tekton-operator-webhook-668b6fdd5d-57qr6 4c2979e1b01aed7a7084fbbf983cf7a307492c7954672d0cb3c6b0d077a64108 network default NAD default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-57qr6 4c2979e1b01aed7a7084fbbf983cf7a307492c7954672d0cb3c6b0d077a64108 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:5e:2d:19:17:0a\",\"name\":\"4c2979e1b01aed7\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e0e89d27-bea9-4c4a-84ba-b5a283047f40\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:20.544813 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=179 W0331 04:02:27.376238 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0331 04:02:32.538465 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=44 I0331 04:02:32.848166 4042 namespace.go:98] [openshift-pipelines] adding namespace I0331 04:02:32.849048 4042 namespace.go:102] [openshift-pipelines] adding namespace took 852.633µs I0331 04:02:32.868215 4042 namespace.go:142] [openshift-pipelines] updating namespace I0331 04:02:33.558379 4042 namespace.go:142] [cert-manager] updating namespace I0331 04:02:33.735114 4042 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:33.735673 4042 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:02:33.756442 4042 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:33.756460 4042 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:02:33.759645 4042 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:02:33.767596 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0331 04:02:33.767616 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0331 04:02:34.161479 4042 namespace.go:142] [konflux-operator] updating namespace I0331 04:02:34.359259 4042 namespace.go:142] [konflux-ui] updating namespace I0331 04:02:34.564009 4042 namespace.go:142] [open-cluster-management-2pcf4qvmb8c8ij76t9r5ocbboorpi5e4] updating namespace I0331 04:02:34.666054 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0331 04:02:34.968010 4042 namespace.go:142] [rhbk-operator] updating namespace I0331 04:02:35.164829 4042 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:02:35.371894 4042 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:02:35.566551 4042 namespace.go:142] [tsf] updating namespace I0331 04:02:35.772060 4042 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:02:35.971097 4042 namespace.go:142] [tsf-tas] updating namespace I0331 04:02:36.171545 4042 namespace.go:142] [tsf-tpa] updating namespace I0331 04:02:36.304616 4042 namespace.go:142] [cert-manager] updating namespace I0331 04:02:36.315140 4042 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:02:36.324572 4042 namespace.go:142] [default] updating namespace I0331 04:02:36.334977 4042 namespace.go:142] [konflux-operator] updating namespace I0331 04:02:36.346000 4042 namespace.go:142] [konflux-ui] updating namespace I0331 04:02:36.356743 4042 namespace.go:142] [open-cluster-management-2pcf4qvmb8c8ij76t9r5ocbboorpi5e4] updating namespace I0331 04:02:36.366032 4042 namespace.go:142] [openshift] updating namespace I0331 04:02:36.375875 4042 namespace.go:142] [rhbk-operator] updating namespace I0331 04:02:36.385174 4042 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:02:36.394887 4042 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:02:36.403241 4042 namespace.go:142] [tsf] updating namespace I0331 04:02:36.413364 4042 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:02:36.430797 4042 namespace.go:142] [tsf-tas] updating namespace I0331 04:02:36.451799 4042 namespace.go:142] [tsf-tpa] updating namespace I0331 04:02:36.558238 4042 namespace.go:142] [cert-manager] updating namespace I0331 04:02:36.653733 4042 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:02:36.754053 4042 namespace.go:142] [default] updating namespace I0331 04:02:36.856852 4042 namespace.go:142] [konflux-operator] updating namespace I0331 04:02:36.951890 4042 namespace.go:142] [konflux-ui] updating namespace I0331 04:02:37.051794 4042 namespace.go:142] [open-cluster-management-2pcf4qvmb8c8ij76t9r5ocbboorpi5e4] updating namespace I0331 04:02:37.150992 4042 namespace.go:142] [openshift] updating namespace I0331 04:02:37.252862 4042 namespace.go:142] [rhbk-operator] updating namespace I0331 04:02:37.354175 4042 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:02:37.451166 4042 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:02:37.553304 4042 namespace.go:142] [tsf] updating namespace I0331 04:02:37.652813 4042 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:02:37.751851 4042 namespace.go:142] [tsf-tas] updating namespace I0331 04:02:37.854808 4042 namespace.go:142] [tsf-tpa] updating namespace I0331 04:02:38.329208 4042 namespace.go:142] [openshift-pipelines] updating namespace W0331 04:02:43.662171 4042 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0331 04:02:44.669002 4042 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0331 04:02:45.925034 4042 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-96v59] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-96v59 for pod on switch ip-10-0-1-243.ec2.internal I0331 04:02:45.925127 4042 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-96v59 I0331 04:02:45.940538 4042 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-96v59] pod update took 15.434223ms I0331 04:02:45.940626 4042 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-96v59] addLogicalPort annotation time took 15.531507ms I0331 04:02:45.942120 4042 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-96v59] addLogicalPort took 17.100939ms, libovsdb time 1.010184ms I0331 04:02:46.080097 4042 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg for pod on switch ip-10-0-1-243.ec2.internal I0331 04:02:46.080184 4042 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg I0331 04:02:46.095350 4042 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg] pod update took 15.189724ms I0331 04:02:46.095370 4042 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg] addLogicalPort annotation time took 15.215095ms I0331 04:02:46.096633 4042 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg] addLogicalPort took 16.551237ms, libovsdb time 892.576µs I0331 04:02:46.243190 4042 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-wmb25] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-7988b548df-wmb25 for pod on switch ip-10-0-1-243.ec2.internal I0331 04:02:46.243296 4042 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-wmb25 I0331 04:02:46.261207 4042 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-wmb25] pod update took 17.947642ms I0331 04:02:46.261234 4042 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-wmb25] addLogicalPort annotation time took 17.978616ms I0331 04:02:46.262507 4042 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-wmb25] addLogicalPort took 19.318071ms, libovsdb time 863.623µs I0331 04:02:46.288543 4042 cni.go:403] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-96v59 64a1c54c5fee98fbb55c773c2c2ad48ca57d7ed326219707459fd3f7fe863d88 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-96v59 64a1c54c5fee98fbb55c773c2c2ad48ca57d7ed326219707459fd3f7fe863d88 network default NAD default] I0331 04:02:46.297637 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-96v59, hostIfaceName: 64a1c54c5fee98f, network: default, NAD default, SandboxID: "64a1c54c5fee98fbb55c773c2c2ad48ca57d7ed326219707459fd3f7fe863d88", PCI device ID: , UID: "c37fe543-f48f-447b-bed0-addfd1e808e5", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0331 04:02:46.341350 4042 cni.go:424] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-96v59 64a1c54c5fee98fbb55c773c2c2ad48ca57d7ed326219707459fd3f7fe863d88 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-96v59 64a1c54c5fee98fbb55c773c2c2ad48ca57d7ed326219707459fd3f7fe863d88 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:2d:6a:13:fe:5f\",\"name\":\"64a1c54c5fee98f\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2bf495f4-7584-407c-9985-5720b9dbf666\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:46.440836 4042 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg 2103a37f8d617ebd2b7aa656cbdaedc078e797ce641cd02f82fc6f8eb30d1bef network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg 2103a37f8d617ebd2b7aa656cbdaedc078e797ce641cd02f82fc6f8eb30d1bef network default NAD default] I0331 04:02:46.448947 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg, hostIfaceName: 2103a37f8d617eb, network: default, NAD default, SandboxID: "2103a37f8d617ebd2b7aa656cbdaedc078e797ce641cd02f82fc6f8eb30d1bef", PCI device ID: , UID: "51965bba-eed3-4abb-b75f-6a0da619246c", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0331 04:02:46.498405 4042 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg 2103a37f8d617ebd2b7aa656cbdaedc078e797ce641cd02f82fc6f8eb30d1bef network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-7k5hg 2103a37f8d617ebd2b7aa656cbdaedc078e797ce641cd02f82fc6f8eb30d1bef network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:9f:c0:6b:85:ea\",\"name\":\"2103a37f8d617eb\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/990b852c-6d50-4938-94a6-f61f878853e1\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:02:46.618184 4042 cni.go:403] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-wmb25 b032c7d1db13ea7ceaf463a17f86b382b17e4a3499645ce954c30330f8681486 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-wmb25 b032c7d1db13ea7ceaf463a17f86b382b17e4a3499645ce954c30330f8681486 network default NAD default] I0331 04:02:46.626965 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-7988b548df-wmb25, hostIfaceName: b032c7d1db13ea7, network: default, NAD default, SandboxID: "b032c7d1db13ea7ceaf463a17f86b382b17e4a3499645ce954c30330f8681486", PCI device ID: , UID: "59d80f69-8347-4daf-8e81-e6d311fcbaf6", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0331 04:02:46.669331 4042 cni.go:424] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-wmb25 b032c7d1db13ea7ceaf463a17f86b382b17e4a3499645ce954c30330f8681486 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-wmb25 b032c7d1db13ea7ceaf463a17f86b382b17e4a3499645ce954c30330f8681486 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:bc:5a:c5:94:8e\",\"name\":\"b032c7d1db13ea7\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/56670704-d115-4cd5-bb21-07cd21027409\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:02:49.210855 4042 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:02:49.210879 4042 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:02:49.210886 4042 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:02:49.210892 4042 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:02:49.698171 4042 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:02:49.698202 4042 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:02:49.698210 4042 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:02:50.774005 4042 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:02:50.774024 4042 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:02:50.774029 4042 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0331 04:02:51.665951 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 W0331 04:03:01.563930 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:03:01.563950 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:03:01.563956 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0331 04:03:10.983658 4042 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-wt7wm] creating logical port openshift-pipelines_tekton-triggers-controller-68564d75ff-wt7wm for pod on switch ip-10-0-1-243.ec2.internal I0331 04:03:10.983767 4042 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-68564d75ff-wt7wm I0331 04:03:11.003134 4042 pod.go:62] [openshift-pipelines/tekton-triggers-controller-68564d75ff-wt7wm] pod update took 19.401119ms I0331 04:03:11.003164 4042 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-wt7wm] addLogicalPort annotation time took 19.436177ms I0331 04:03:11.004584 4042 pods.go:271] [openshift-pipelines/tekton-triggers-controller-68564d75ff-wt7wm] addLogicalPort took 20.947461ms, libovsdb time 902.587µs I0331 04:03:11.196685 4042 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-hhnm5] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-8bb674894-hhnm5 for pod on switch ip-10-0-1-243.ec2.internal I0331 04:03:11.196765 4042 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-hhnm5 I0331 04:03:11.212265 4042 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-hhnm5] pod update took 15.520413ms I0331 04:03:11.212284 4042 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-hhnm5] addLogicalPort annotation time took 15.54527ms I0331 04:03:11.213552 4042 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-hhnm5] addLogicalPort took 16.881805ms, libovsdb time 896µs I0331 04:03:11.344397 4042 cni.go:403] [openshift-pipelines/tekton-triggers-controller-68564d75ff-wt7wm ba21d84356a4af6a7ebdf010a90b2d135eb33f46eef7a76447bb55ec4583926a network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-wt7wm ba21d84356a4af6a7ebdf010a90b2d135eb33f46eef7a76447bb55ec4583926a network default NAD default] I0331 04:03:11.353536 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-68564d75ff-wt7wm, hostIfaceName: ba21d84356a4af6, network: default, NAD default, SandboxID: "ba21d84356a4af6a7ebdf010a90b2d135eb33f46eef7a76447bb55ec4583926a", PCI device ID: , UID: "a5797fb4-0ebf-448a-b1e2-139907d5c1f8", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0331 04:03:11.398931 4042 cni.go:424] [openshift-pipelines/tekton-triggers-controller-68564d75ff-wt7wm ba21d84356a4af6a7ebdf010a90b2d135eb33f46eef7a76447bb55ec4583926a network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-wt7wm ba21d84356a4af6a7ebdf010a90b2d135eb33f46eef7a76447bb55ec4583926a network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:d2:7b:bc:6d:db\",\"name\":\"ba21d84356a4af6\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7d51140c-17f3-4f12-8f19-0d058237bb9e\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:03:11.554662 4042 cni.go:403] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-hhnm5 660ebc925ce3dc240abcefbc6e7ac892a0cccb2f2176ee964e4c4926e4d34f3e network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-hhnm5 660ebc925ce3dc240abcefbc6e7ac892a0cccb2f2176ee964e4c4926e4d34f3e network default NAD default] I0331 04:03:11.562984 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-8bb674894-hhnm5, hostIfaceName: 660ebc925ce3dc2, network: default, NAD default, SandboxID: "660ebc925ce3dc240abcefbc6e7ac892a0cccb2f2176ee964e4c4926e4d34f3e", PCI device ID: , UID: "31279958-71b9-4427-b603-0f8e355d3a74", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0331 04:03:11.607484 4042 cni.go:424] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-hhnm5 660ebc925ce3dc240abcefbc6e7ac892a0cccb2f2176ee964e4c4926e4d34f3e network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-hhnm5 660ebc925ce3dc240abcefbc6e7ac892a0cccb2f2176ee964e4c4926e4d34f3e network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:98:e5:b0:65:0d\",\"name\":\"660ebc925ce3dc2\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/19b90c0d-5db3-46ee-92ac-b7bdc90f51a4\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:03:13.708275 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0331 04:03:13.975491 4042 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:13.976067 4042 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:13.996913 4042 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:13.996938 4042 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:14.008701 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0331 04:03:14.008723 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure W0331 04:03:20.857203 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:20.857225 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:20.859629 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0331 04:03:24.682608 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0331 04:03:28.635620 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0331 04:03:31.312662 4042 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0331 04:03:39.189836 4042 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:39.190362 4042 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:03:39.947624 4042 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:39.947649 4042 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:03:39.955225 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0331 04:03:39.955256 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0331 04:03:41.972338 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:41.972360 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:03:41.974969 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0331 04:03:54.247696 4042 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-j47hp] creating logical port openshift-pipelines_tekton-results-watcher-6f946f97cf-j47hp for pod on switch ip-10-0-1-243.ec2.internal I0331 04:03:54.247798 4042 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-6f946f97cf-j47hp I0331 04:03:54.263217 4042 pod.go:62] [openshift-pipelines/tekton-results-watcher-6f946f97cf-j47hp] pod update took 15.449025ms I0331 04:03:54.263240 4042 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-watcher-6f946f97cf-j47hp] addLogicalPort annotation time took 15.476357ms I0331 04:03:54.264441 4042 pods.go:271] [openshift-pipelines/tekton-results-watcher-6f946f97cf-j47hp] addLogicalPort took 16.760094ms, libovsdb time 812.637µs I0331 04:03:54.537187 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=48 I0331 04:03:54.611037 4042 cni.go:403] [openshift-pipelines/tekton-results-watcher-6f946f97cf-j47hp 646096624fe1c9b07149fd7b2aca255481a54ed5a137b70878cb02d79ed125d1 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-j47hp 646096624fe1c9b07149fd7b2aca255481a54ed5a137b70878cb02d79ed125d1 network default NAD default] I0331 04:03:54.619985 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-6f946f97cf-j47hp, hostIfaceName: 646096624fe1c9b, network: default, NAD default, SandboxID: "646096624fe1c9b07149fd7b2aca255481a54ed5a137b70878cb02d79ed125d1", PCI device ID: , UID: "f99c86c4-ef31-4acd-a168-75c15d93f0dd", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0331 04:03:54.665328 4042 cni.go:424] [openshift-pipelines/tekton-results-watcher-6f946f97cf-j47hp 646096624fe1c9b07149fd7b2aca255481a54ed5a137b70878cb02d79ed125d1 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-6f946f97cf-j47hp 646096624fe1c9b07149fd7b2aca255481a54ed5a137b70878cb02d79ed125d1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:91:d0:1e:fb:b2\",\"name\":\"646096624fe1c9b\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe3b5431-323c-47a6-887d-a4822174c2bf\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:04:01.615842 4042 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:04:01.615864 4042 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:04:01.615869 4042 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0331 04:04:08.630990 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0331 04:04:09.268730 4042 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:09.269271 4042 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:09.317298 4042 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:09.317322 4042 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:09.325406 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0331 04:04:09.325426 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0331 04:04:11.149567 4042 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-n27wm-xs9fs] creating logical port tsf-tas_segment-backup-installation-n27wm-xs9fs for pod on switch ip-10-0-1-243.ec2.internal I0331 04:04:11.149654 4042 kube.go:256] Updating pod tsf-tas/segment-backup-installation-n27wm-xs9fs I0331 04:04:11.162717 4042 pod.go:62] [tsf-tas/segment-backup-installation-n27wm-xs9fs] pod update took 13.085821ms I0331 04:04:11.162738 4042 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-n27wm-xs9fs] addLogicalPort annotation time took 13.112721ms I0331 04:04:11.164021 4042 pods.go:271] [tsf-tas/segment-backup-installation-n27wm-xs9fs] addLogicalPort took 14.468873ms, libovsdb time 874.115µs I0331 04:04:11.520739 4042 cni.go:403] [tsf-tas/segment-backup-installation-n27wm-xs9fs 3765f5052868aa6ce3adad979bfb9bf1291e28558a9ec5e20033742607b0cce6 network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-n27wm-xs9fs 3765f5052868aa6ce3adad979bfb9bf1291e28558a9ec5e20033742607b0cce6 network default NAD default] I0331 04:04:11.529860 4042 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-n27wm-xs9fs, hostIfaceName: 3765f5052868aa6, network: default, NAD default, SandboxID: "3765f5052868aa6ce3adad979bfb9bf1291e28558a9ec5e20033742607b0cce6", PCI device ID: , UID: "605610ad-a048-4805-8d4b-8ddb9251fab0", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0331 04:04:11.577904 4042 cni.go:424] [tsf-tas/segment-backup-installation-n27wm-xs9fs 3765f5052868aa6ce3adad979bfb9bf1291e28558a9ec5e20033742607b0cce6 network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-n27wm-xs9fs 3765f5052868aa6ce3adad979bfb9bf1291e28558a9ec5e20033742607b0cce6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ee:34:08:4c:fe:90\",\"name\":\"3765f5052868aa6\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba7387ac-005e-49f8-965f-8ca3ff5f235f\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:04:12.290430 4042 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-z5x6w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:12.291088 4042 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-z5x6w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:12.294948 4042 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-589cf6b794-6d7n8] creating logical port tsf-tas_fulcio-server-589cf6b794-6d7n8 for pod on switch ip-10-0-1-243.ec2.internal I0331 04:04:12.295045 4042 kube.go:256] Updating pod tsf-tas/fulcio-server-589cf6b794-6d7n8 I0331 04:04:12.327052 4042 pod.go:62] [tsf-tas/fulcio-server-589cf6b794-6d7n8] pod update took 32.030209ms I0331 04:04:12.327142 4042 base_network_controller_pods.go:937] [default/tsf-tas/fulcio-server-589cf6b794-6d7n8] addLogicalPort annotation time took 32.12347ms I0331 04:04:12.328807 4042 pods.go:271] [tsf-tas/fulcio-server-589cf6b794-6d7n8] addLogicalPort took 33.875669ms, libovsdb time 1.065981ms I0331 04:04:12.670890 4042 cni.go:403] [tsf-tas/fulcio-server-589cf6b794-6d7n8 441b2239e535e939be2e8ea8d523b682c0138d6a282eda6d8c63c4261f8380c5 network default NAD default] ADD starting CNI request [tsf-tas/fulcio-server-589cf6b794-6d7n8 441b2239e535e939be2e8ea8d523b682c0138d6a282eda6d8c63c4261f8380c5 network default NAD default] I0331 04:04:12.681991 4042 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-589cf6b794-6d7n8, hostIfaceName: 441b2239e535e93, network: default, NAD default, SandboxID: "441b2239e535e939be2e8ea8d523b682c0138d6a282eda6d8c63c4261f8380c5", PCI device ID: , UID: "1f43c1ec-86c0-4125-a0b3-ea5c11056859", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0331 04:04:12.734616 4042 cni.go:424] [tsf-tas/fulcio-server-589cf6b794-6d7n8 441b2239e535e939be2e8ea8d523b682c0138d6a282eda6d8c63c4261f8380c5 network default NAD default] ADD finished CNI request [tsf-tas/fulcio-server-589cf6b794-6d7n8 441b2239e535e939be2e8ea8d523b682c0138d6a282eda6d8c63c4261f8380c5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:77:4e:f4:ec:11\",\"name\":\"441b2239e535e93\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a6ddbc84-5b0b-4635-ac10-ece28006403b\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:04:13.320881 4042 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-z5x6w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:13.320911 4042 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-z5x6w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:13.667488 4042 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:13.668190 4042 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:14.160763 4042 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:14.160798 4042 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:14.169717 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0331 04:04:14.169746 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0331 04:04:15.001693 4042 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:04:15.001716 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:04:16.198912 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:04:16.198934 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:04:16.200120 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0331 04:04:20.216652 4042 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0331 04:04:20.248279 4042 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0331 04:04:26.020906 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:04:26.020931 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:04:28.264713 4042 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0331 04:04:31.094158 4042 cni.go:403] [tsf-tas/segment-backup-installation-n27wm-xs9fs 3765f5052868aa6ce3adad979bfb9bf1291e28558a9ec5e20033742607b0cce6 network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-n27wm-xs9fs 3765f5052868aa6ce3adad979bfb9bf1291e28558a9ec5e20033742607b0cce6 network default NAD default] I0331 04:04:31.182231 4042 cni.go:424] [tsf-tas/segment-backup-installation-n27wm-xs9fs 3765f5052868aa6ce3adad979bfb9bf1291e28558a9ec5e20033742607b0cce6 network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-n27wm-xs9fs 3765f5052868aa6ce3adad979bfb9bf1291e28558a9ec5e20033742607b0cce6 network default NAD default], result "{}", err I0331 04:04:31.247888 4042 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-n27wm-xs9fs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:31.247926 4042 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-n27wm-xs9fs I0331 04:04:31.249196 4042 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-n27wm-xs9fs, ips: 10.128.0.39 I0331 04:04:31.249229 4042 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-n27wm-xs9fs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:04:32.110925 4042 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-n27wm-xs9fs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:32.110949 4042 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-n27wm-xs9fs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:04:55.532391 4042 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-z5n5w] creating logical port openshift-pipelines_pipelines-as-code-controller-79b6fbbc5c-z5n5w for pod on switch ip-10-0-1-243.ec2.internal I0331 04:04:55.532591 4042 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-z5n5w I0331 04:04:55.549367 4042 pod.go:62] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-z5n5w] pod update took 16.811645ms I0331 04:04:55.549391 4042 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-z5n5w] addLogicalPort annotation time took 16.841812ms I0331 04:04:55.550840 4042 pods.go:271] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-z5n5w] addLogicalPort took 18.467609ms, libovsdb time 899.077µs I0331 04:04:55.641390 4042 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8zgf4] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-8zgf4 for pod on switch ip-10-0-1-243.ec2.internal I0331 04:04:55.641473 4042 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8zgf4 I0331 04:04:55.656705 4042 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8zgf4] pod update took 15.251281ms I0331 04:04:55.656726 4042 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8zgf4] addLogicalPort annotation time took 15.278647ms I0331 04:04:55.657995 4042 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8zgf4] addLogicalPort took 16.619631ms, libovsdb time 837.579µs I0331 04:04:56.091190 4042 cni.go:403] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8zgf4 70b6dd17883c67f4233f0f54c3d4f26a449f894305063ee4be7974a4c92bfcff network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8zgf4 70b6dd17883c67f4233f0f54c3d4f26a449f894305063ee4be7974a4c92bfcff network default NAD default] I0331 04:04:56.099523 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-8zgf4, hostIfaceName: 70b6dd17883c67f, network: default, NAD default, SandboxID: "70b6dd17883c67f4233f0f54c3d4f26a449f894305063ee4be7974a4c92bfcff", PCI device ID: , UID: "33d53a36-8fa0-433f-b898-8a936a962d4f", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0331 04:04:56.145476 4042 cni.go:424] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8zgf4 70b6dd17883c67f4233f0f54c3d4f26a449f894305063ee4be7974a4c92bfcff network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-8zgf4 70b6dd17883c67f4233f0f54c3d4f26a449f894305063ee4be7974a4c92bfcff network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:a6:42:50:cc:1d\",\"name\":\"70b6dd17883c67f\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7d7cf3a6-54eb-4ada-839f-0063ab8a1385\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:04:56.355601 4042 cni.go:403] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-z5n5w 8dc8ca036e76c2028dfa1ab39c897890c9823de19227eb6434139f42b0f04d4b network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-z5n5w 8dc8ca036e76c2028dfa1ab39c897890c9823de19227eb6434139f42b0f04d4b network default NAD default] I0331 04:04:56.364347 4042 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-controller-79b6fbbc5c-z5n5w, hostIfaceName: 8dc8ca036e76c20, network: default, NAD default, SandboxID: "8dc8ca036e76c2028dfa1ab39c897890c9823de19227eb6434139f42b0f04d4b", PCI device ID: , UID: "20f939ef-8e90-4299-a210-d223236936c4", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0331 04:04:56.409168 4042 cni.go:424] [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-z5n5w 8dc8ca036e76c2028dfa1ab39c897890c9823de19227eb6434139f42b0f04d4b network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-controller-79b6fbbc5c-z5n5w 8dc8ca036e76c2028dfa1ab39c897890c9823de19227eb6434139f42b0f04d4b network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:d9:87:4a:b6:dc\",\"name\":\"8dc8ca036e76c20\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/479ef6b5-0744-4ba6-b1af-d011efbc030b\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:04:56.653051 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 W0331 04:04:59.246114 4042 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0331 04:05:00.255552 4042 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:05:00.255575 4042 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:05:00.283695 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8082: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:05:00.283717 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0331 04:05:05.659419 4042 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0331 04:05:07.602234 4042 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:07.602809 4042 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:07.622950 4042 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:07.622967 4042 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:07.631454 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0331 04:05:07.631478 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0331 04:05:08.094170 4042 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-n7pnj-c765d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:08.094757 4042 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-n7pnj-c765d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:08.984238 4042 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-n7pnj-c765d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:08.984262 4042 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-n7pnj-c765d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:10.136692 4042 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-mpf86-tjlf6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:10.137266 4042 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-mpf86-tjlf6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0331 04:05:10.264741 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0331 04:05:11.002440 4042 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-mpf86-tjlf6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:11.002465 4042 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-mpf86-tjlf6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:12.597211 4042 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-85588889dc-kn9bm] creating logical port tsf-tas_ctlog-85588889dc-kn9bm for pod on switch ip-10-0-1-243.ec2.internal I0331 04:05:12.597372 4042 kube.go:256] Updating pod tsf-tas/ctlog-85588889dc-kn9bm I0331 04:05:12.614043 4042 pod.go:62] [tsf-tas/ctlog-85588889dc-kn9bm] pod update took 16.691806ms I0331 04:05:12.614070 4042 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-85588889dc-kn9bm] addLogicalPort annotation time took 16.724114ms I0331 04:05:12.615484 4042 pods.go:271] [tsf-tas/ctlog-85588889dc-kn9bm] addLogicalPort took 18.299802ms, libovsdb time 852.858µs I0331 04:05:12.956818 4042 cni.go:403] [tsf-tas/ctlog-85588889dc-kn9bm 01cf9fd12c4656c1bb9defed7d1197eab15c93404c775ccdfd8c79a298b4056d network default NAD default] ADD starting CNI request [tsf-tas/ctlog-85588889dc-kn9bm 01cf9fd12c4656c1bb9defed7d1197eab15c93404c775ccdfd8c79a298b4056d network default NAD default] I0331 04:05:12.965243 4042 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-85588889dc-kn9bm, hostIfaceName: 01cf9fd12c4656c, network: default, NAD default, SandboxID: "01cf9fd12c4656c1bb9defed7d1197eab15c93404c775ccdfd8c79a298b4056d", PCI device ID: , UID: "0c1d7b3b-65b5-434f-8719-68f7cd0794c8", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0331 04:05:13.012462 4042 cni.go:424] [tsf-tas/ctlog-85588889dc-kn9bm 01cf9fd12c4656c1bb9defed7d1197eab15c93404c775ccdfd8c79a298b4056d network default NAD default] ADD finished CNI request [tsf-tas/ctlog-85588889dc-kn9bm 01cf9fd12c4656c1bb9defed7d1197eab15c93404c775ccdfd8c79a298b4056d network default NAD default], result "{\"interfaces\":[{\"mac\":\"42:35:c6:e2:c7:db\",\"name\":\"01cf9fd12c4656c\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1917720f-4c03-4bc3-b320-aeea3f5d3693\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:05:14.243894 4042 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-8c6979544-2ssrb] creating logical port tsf-tas_rekor-redis-8c6979544-2ssrb for pod on switch ip-10-0-1-243.ec2.internal I0331 04:05:14.244004 4042 kube.go:256] Updating pod tsf-tas/rekor-redis-8c6979544-2ssrb I0331 04:05:14.257544 4042 pod.go:62] [tsf-tas/rekor-redis-8c6979544-2ssrb] pod update took 13.568195ms I0331 04:05:14.257565 4042 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-8c6979544-2ssrb] addLogicalPort annotation time took 13.595677ms I0331 04:05:14.258973 4042 pods.go:271] [tsf-tas/rekor-redis-8c6979544-2ssrb] addLogicalPort took 15.100048ms, libovsdb time 955.818µs I0331 04:05:14.421518 4042 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-f7889fdb4-zjbp4] creating logical port tsf-tas_rekor-search-ui-f7889fdb4-zjbp4 for pod on switch ip-10-0-1-243.ec2.internal I0331 04:05:14.421608 4042 kube.go:256] Updating pod tsf-tas/rekor-search-ui-f7889fdb4-zjbp4 I0331 04:05:14.439005 4042 pod.go:62] [tsf-tas/rekor-search-ui-f7889fdb4-zjbp4] pod update took 17.419414ms I0331 04:05:14.439033 4042 base_network_controller_pods.go:937] [default/tsf-tas/rekor-search-ui-f7889fdb4-zjbp4] addLogicalPort annotation time took 17.450656ms I0331 04:05:14.440563 4042 pods.go:271] [tsf-tas/rekor-search-ui-f7889fdb4-zjbp4] addLogicalPort took 19.083613ms, libovsdb time 918.141µs I0331 04:05:14.785585 4042 cni.go:403] [tsf-tas/rekor-search-ui-f7889fdb4-zjbp4 9a0ea860c2cdf709a3067f5267b77fddcda2c4e3a8f1805af4d2382989cbaa04 network default NAD default] ADD starting CNI request [tsf-tas/rekor-search-ui-f7889fdb4-zjbp4 9a0ea860c2cdf709a3067f5267b77fddcda2c4e3a8f1805af4d2382989cbaa04 network default NAD default] I0331 04:05:14.794028 4042 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-f7889fdb4-zjbp4, hostIfaceName: 9a0ea860c2cdf70, network: default, NAD default, SandboxID: "9a0ea860c2cdf709a3067f5267b77fddcda2c4e3a8f1805af4d2382989cbaa04", PCI device ID: , UID: "160669e9-283c-4783-b850-7b10f95d3d11", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0331 04:05:14.839688 4042 cni.go:424] [tsf-tas/rekor-search-ui-f7889fdb4-zjbp4 9a0ea860c2cdf709a3067f5267b77fddcda2c4e3a8f1805af4d2382989cbaa04 network default NAD default] ADD finished CNI request [tsf-tas/rekor-search-ui-f7889fdb4-zjbp4 9a0ea860c2cdf709a3067f5267b77fddcda2c4e3a8f1805af4d2382989cbaa04 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:2b:66:8d:dc:8e\",\"name\":\"9a0ea860c2cdf70\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a0d0a8f0-d7f3-44a9-b67b-4ec56b76f486\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:05:15.206809 4042 cni.go:403] [tsf-tas/rekor-redis-8c6979544-2ssrb 474a03ba2c2caf946871ab601d1e35cd881ef73fda90d23f236e72cea07e4ea6 network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-8c6979544-2ssrb 474a03ba2c2caf946871ab601d1e35cd881ef73fda90d23f236e72cea07e4ea6 network default NAD default] I0331 04:05:15.215422 4042 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-8c6979544-2ssrb, hostIfaceName: 474a03ba2c2caf9, network: default, NAD default, SandboxID: "474a03ba2c2caf946871ab601d1e35cd881ef73fda90d23f236e72cea07e4ea6", PCI device ID: , UID: "10dd1380-27a4-42dd-9b08-28fab67fd58e", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0331 04:05:15.261178 4042 cni.go:424] [tsf-tas/rekor-redis-8c6979544-2ssrb 474a03ba2c2caf946871ab601d1e35cd881ef73fda90d23f236e72cea07e4ea6 network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-8c6979544-2ssrb 474a03ba2c2caf946871ab601d1e35cd881ef73fda90d23f236e72cea07e4ea6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:f2:21:7d:ce:9c\",\"name\":\"474a03ba2c2caf9\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d0913c0b-c9e3-492d-bb04-91bd16c5d87e\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:05:15.337417 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0331 04:05:16.159911 4042 obj_retry.go:464] Detected object tsf/patch-tekton-config-hsjj6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:16.160782 4042 obj_retry.go:464] Detected object tsf/patch-tekton-config-hsjj6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:17.183121 4042 obj_retry.go:458] Detected object tsf/patch-tekton-config-hsjj6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:17.183152 4042 obj_retry.go:458] Detected object tsf/patch-tekton-config-hsjj6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0331 04:05:18.356537 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0331 04:05:18.385002 4042 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0331 04:05:21.642019 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0331 04:05:25.230845 4042 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-5x4zw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:25.231390 4042 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-5x4zw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:26.256889 4042 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-5x4zw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:26.256913 4042 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-5x4zw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:26.300151 4042 obj_retry.go:458] Detected object tsf/patch-tekton-config-hsjj6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:26.300174 4042 obj_retry.go:458] Detected object tsf/patch-tekton-config-hsjj6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:26.305859 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-hsjj6 I0331 04:05:26.305882 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-hsjj6 W0331 04:05:26.350966 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog W0331 04:05:28.575875 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0331 04:05:29.647441 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 W0331 04:05:30.382478 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui W0331 04:05:30.410758 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0331 04:05:36.294478 4042 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:36.295040 4042 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:36.312118 4042 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:36.312138 4042 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:36.320966 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0331 04:05:36.320988 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0331 04:05:40.780102 4042 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:40.780641 4042 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:40.800590 4042 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:40.800609 4042 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:40.809468 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0331 04:05:40.809489 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0331 04:05:47.394967 4042 obj_retry.go:464] Detected object tsf-tpa/create-db-gkhp4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:47.395600 4042 obj_retry.go:464] Detected object tsf-tpa/create-db-gkhp4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:48.423534 4042 obj_retry.go:458] Detected object tsf-tpa/create-db-gkhp4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:48.423558 4042 obj_retry.go:458] Detected object tsf-tpa/create-db-gkhp4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:51.392249 4042 obj_retry.go:464] Detected object tsf-tpa/migrate-db-sdrs7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:51.392809 4042 obj_retry.go:464] Detected object tsf-tpa/migrate-db-sdrs7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:52.407091 4042 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-c24lk-4p6qx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:52.407638 4042 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-c24lk-4p6qx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:52.415661 4042 obj_retry.go:458] Detected object tsf-tpa/migrate-db-sdrs7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:52.415683 4042 obj_retry.go:458] Detected object tsf-tpa/migrate-db-sdrs7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:53.280663 4042 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-c24lk-4p6qx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:53.280685 4042 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-c24lk-4p6qx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:54.412312 4042 obj_retry.go:464] Detected object tsf-tpa/create-importers-8bsnv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:54.412872 4042 obj_retry.go:464] Detected object tsf-tpa/create-importers-8bsnv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:05:55.439424 4042 obj_retry.go:458] Detected object tsf-tpa/create-importers-8bsnv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:55.439446 4042 obj_retry.go:458] Detected object tsf-tpa/create-importers-8bsnv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:05:55.563517 4042 base_network_controller_pods.go:487] [default/tsf-tpa/server-6855569646-9vpzw] creating logical port tsf-tpa_server-6855569646-9vpzw for pod on switch ip-10-0-1-243.ec2.internal I0331 04:05:55.563611 4042 kube.go:256] Updating pod tsf-tpa/server-6855569646-9vpzw I0331 04:05:55.582830 4042 pod.go:62] [tsf-tpa/server-6855569646-9vpzw] pod update took 19.243431ms I0331 04:05:55.582852 4042 base_network_controller_pods.go:937] [default/tsf-tpa/server-6855569646-9vpzw] addLogicalPort annotation time took 19.270094ms I0331 04:05:55.583057 4042 base_network_controller_pods.go:487] [default/tsf-tpa/importer-74cd48f5c7-khxrc] creating logical port tsf-tpa_importer-74cd48f5c7-khxrc for pod on switch ip-10-0-1-243.ec2.internal I0331 04:05:55.583161 4042 kube.go:256] Updating pod tsf-tpa/importer-74cd48f5c7-khxrc I0331 04:05:55.584373 4042 pods.go:271] [tsf-tpa/server-6855569646-9vpzw] addLogicalPort took 20.889711ms, libovsdb time 970.989µs I0331 04:05:55.597108 4042 pod.go:62] [tsf-tpa/importer-74cd48f5c7-khxrc] pod update took 13.982733ms I0331 04:05:55.597126 4042 base_network_controller_pods.go:937] [default/tsf-tpa/importer-74cd48f5c7-khxrc] addLogicalPort annotation time took 14.005932ms I0331 04:05:55.598259 4042 pods.go:271] [tsf-tpa/importer-74cd48f5c7-khxrc] addLogicalPort took 15.218676ms, libovsdb time 719.003µs I0331 04:05:56.270008 4042 namespace.go:142] [openshift-pipelines] updating namespace I0331 04:05:57.540138 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=134 I0331 04:05:58.625247 4042 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0331 04:06:03.734016 4042 cni.go:403] [tsf-tpa/server-6855569646-9vpzw a0e7355367d4500444425c7b92d4532786bfab1e96e56a2e19d6532854b6c859 network default NAD default] ADD starting CNI request [tsf-tpa/server-6855569646-9vpzw a0e7355367d4500444425c7b92d4532786bfab1e96e56a2e19d6532854b6c859 network default NAD default] I0331 04:06:03.742931 4042 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: server-6855569646-9vpzw, hostIfaceName: a0e7355367d4500, network: default, NAD default, SandboxID: "a0e7355367d4500444425c7b92d4532786bfab1e96e56a2e19d6532854b6c859", PCI device ID: , UID: "fc2ed039-3627-47eb-9bdd-0a3c0fc5a631", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0331 04:06:03.789271 4042 cni.go:424] [tsf-tpa/server-6855569646-9vpzw a0e7355367d4500444425c7b92d4532786bfab1e96e56a2e19d6532854b6c859 network default NAD default] ADD finished CNI request [tsf-tpa/server-6855569646-9vpzw a0e7355367d4500444425c7b92d4532786bfab1e96e56a2e19d6532854b6c859 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:05:17:48:d9:14\",\"name\":\"a0e7355367d4500\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/123aa9cf-9726-4b15-8b4b-70f95765c11f\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0331 04:06:04.041926 4042 cni.go:403] [tsf-tpa/importer-74cd48f5c7-khxrc f9870d991838471066a00d7f3543ebed7dee9d388f3d895012e969a252b5fd62 network default NAD default] ADD starting CNI request [tsf-tpa/importer-74cd48f5c7-khxrc f9870d991838471066a00d7f3543ebed7dee9d388f3d895012e969a252b5fd62 network default NAD default] I0331 04:06:04.051518 4042 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: importer-74cd48f5c7-khxrc, hostIfaceName: f9870d991838471, network: default, NAD default, SandboxID: "f9870d991838471066a00d7f3543ebed7dee9d388f3d895012e969a252b5fd62", PCI device ID: , UID: "90a3ecb0-d920-4bc9-9f94-d92d35e30934", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0331 04:06:04.303868 4042 cni.go:424] [tsf-tpa/importer-74cd48f5c7-khxrc f9870d991838471066a00d7f3543ebed7dee9d388f3d895012e969a252b5fd62 network default NAD default] ADD finished CNI request [tsf-tpa/importer-74cd48f5c7-khxrc f9870d991838471066a00d7f3543ebed7dee9d388f3d895012e969a252b5fd62 network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:ce:32:54:3e:25\",\"name\":\"f9870d991838471\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cbae81db-afda-4db6-afbe-da4ebdffc486\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0331 04:06:06.637368 4042 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0331 04:06:08.359067 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0331 04:06:17.647553 4042 util.go:959] Failed to build node endpoints for node ip-10-0-1-243.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0331 04:06:35.065024 4042 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:35.065617 4042 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:35.112524 4042 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:35.112573 4042 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:35.122868 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0331 04:06:35.122893 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0331 04:06:35.866387 4042 namespace.go:98] [build-service] adding namespace I0331 04:06:35.867339 4042 namespace.go:102] [build-service] adding namespace took 929.223µs I0331 04:06:35.872434 4042 namespace.go:98] [image-controller] adding namespace I0331 04:06:35.872538 4042 namespace.go:98] [integration-service] adding namespace I0331 04:06:35.873126 4042 namespace.go:102] [image-controller] adding namespace took 674.55µs I0331 04:06:35.873675 4042 namespace.go:102] [integration-service] adding namespace took 1.122208ms I0331 04:06:35.899518 4042 namespace.go:142] [build-service] updating namespace I0331 04:06:35.929294 4042 namespace.go:142] [build-service] updating namespace I0331 04:06:35.937854 4042 namespace.go:142] [image-controller] updating namespace I0331 04:06:35.976007 4042 namespace.go:142] [image-controller] updating namespace I0331 04:06:35.979967 4042 namespace.go:142] [integration-service] updating namespace I0331 04:06:36.035921 4042 namespace.go:142] [integration-service] updating namespace I0331 04:06:36.144151 4042 namespace.go:142] [build-service] updating namespace I0331 04:06:36.171878 4042 namespace.go:142] [integration-service] updating namespace I0331 04:06:36.237872 4042 namespace.go:98] [release-service] adding namespace I0331 04:06:36.238768 4042 namespace.go:102] [release-service] adding namespace took 880.136µs I0331 04:06:36.270369 4042 namespace.go:142] [konflux-ui] updating namespace I0331 04:06:36.281896 4042 namespace.go:142] [release-service] updating namespace I0331 04:06:36.314056 4042 namespace.go:142] [release-service] updating namespace I0331 04:06:36.404670 4042 namespace.go:98] [konflux-info] adding namespace I0331 04:06:36.405622 4042 namespace.go:102] [konflux-info] adding namespace took 922.944µs I0331 04:06:36.448164 4042 namespace.go:98] [namespace-lister] adding namespace I0331 04:06:36.448996 4042 namespace.go:102] [namespace-lister] adding namespace took 789.755µs I0331 04:06:36.473191 4042 namespace.go:142] [konflux-info] updating namespace I0331 04:06:36.544792 4042 namespace.go:142] [konflux-info] updating namespace I0331 04:06:36.559277 4042 namespace.go:98] [enterprise-contract-service] adding namespace I0331 04:06:36.560021 4042 namespace.go:102] [enterprise-contract-service] adding namespace took 728.563µs I0331 04:06:36.592394 4042 namespace.go:142] [namespace-lister] updating namespace I0331 04:06:36.592406 4042 namespace.go:142] [image-controller] updating namespace I0331 04:06:36.755927 4042 namespace.go:142] [namespace-lister] updating namespace I0331 04:06:36.807205 4042 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:06:36.893121 4042 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:06:36.921771 4042 namespace.go:98] [default-tenant] adding namespace I0331 04:06:36.922667 4042 namespace.go:102] [default-tenant] adding namespace took 876.297µs I0331 04:06:36.999550 4042 namespace.go:98] [segment-bridge] adding namespace I0331 04:06:37.000488 4042 namespace.go:102] [segment-bridge] adding namespace took 916.669µs I0331 04:06:37.004871 4042 namespace.go:142] [build-service] updating namespace I0331 04:06:37.072027 4042 namespace.go:142] [default-tenant] updating namespace I0331 04:06:37.148717 4042 namespace.go:142] [default-tenant] updating namespace I0331 04:06:37.187364 4042 namespace.go:142] [segment-bridge] updating namespace I0331 04:06:37.274171 4042 namespace.go:142] [segment-bridge] updating namespace I0331 04:06:37.288874 4042 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0331 04:06:37.289554 4042 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0331 04:06:37.289574 4042 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0331 04:06:37.292248 4042 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0331 04:06:37.340935 4042 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0331 04:06:37.340957 4042 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0331 04:06:37.341691 4042 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0331 04:06:37.867815 4042 namespace.go:142] [image-controller] updating namespace I0331 04:06:38.402452 4042 namespace.go:142] [integration-service] updating namespace I0331 04:06:38.621331 4042 namespace.go:142] [release-service] updating namespace I0331 04:06:38.727640 4042 namespace.go:142] [build-service] updating namespace I0331 04:06:38.743977 4042 namespace.go:142] [image-controller] updating namespace I0331 04:06:38.744099 4042 namespace.go:142] [integration-service] updating namespace I0331 04:06:38.768206 4042 namespace.go:142] [release-service] updating namespace I0331 04:06:38.900191 4042 namespace.go:142] [build-service] updating namespace I0331 04:06:38.998886 4042 namespace.go:142] [image-controller] updating namespace I0331 04:06:39.080665 4042 namespace.go:142] [integration-service] updating namespace I0331 04:06:39.148830 4042 namespace.go:142] [release-service] updating namespace I0331 04:06:39.680039 4042 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-d9l5z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:39.680709 4042 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-d9l5z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:39.749426 4042 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-d9l5z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:39.749450 4042 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-d9l5z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:40.366919 4042 namespace.go:142] [default-tenant] updating namespace I0331 04:06:40.579886 4042 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:06:40.756348 4042 namespace.go:142] [konflux-info] updating namespace I0331 04:06:40.959554 4042 namespace.go:142] [namespace-lister] updating namespace I0331 04:06:41.162100 4042 namespace.go:142] [segment-bridge] updating namespace I0331 04:06:41.289564 4042 namespace.go:142] [default-tenant] updating namespace I0331 04:06:41.304054 4042 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:06:41.317311 4042 namespace.go:142] [konflux-info] updating namespace I0331 04:06:41.331830 4042 namespace.go:142] [namespace-lister] updating namespace I0331 04:06:41.347360 4042 namespace.go:142] [segment-bridge] updating namespace I0331 04:06:41.396651 4042 namespace.go:142] [default-tenant] updating namespace I0331 04:06:41.471707 4042 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:06:41.569265 4042 namespace.go:142] [konflux-info] updating namespace I0331 04:06:41.667879 4042 namespace.go:142] [namespace-lister] updating namespace I0331 04:06:41.775558 4042 namespace.go:142] [segment-bridge] updating namespace W0331 04:06:41.972523 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0331 04:06:42.588310 4042 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0331 04:06:42.615092 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0331 04:06:48.638237 4042 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0331 04:06:51.685917 4042 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0331 04:06:51.685944 4042 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0331 04:06:53.675779 4042 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:06:53.677442 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:06:55.054129 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:06:55.054750 4042 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:06:55.083190 4042 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0331 04:06:55.083434 4042 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0331 04:06:55.119249 4042 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0331 04:06:55.119264 4042 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0331 04:07:05.017911 4042 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-sp2hr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:05.018458 4042 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-sp2hr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:05.120220 4042 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-sp2hr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:05.120240 4042 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-sp2hr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:05.134189 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-sp2hr I0331 04:07:05.134231 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-sp2hr I0331 04:07:59.540964 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=59 I0331 04:08:01.463997 4042 obj_retry.go:464] Detected object konflux-ui/proxy-86995756bd-q2bm6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:01.464621 4042 obj_retry.go:464] Detected object konflux-ui/proxy-86995756bd-q2bm6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:02.089063 4042 obj_retry.go:458] Detected object konflux-ui/proxy-86995756bd-q2bm6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:02.089086 4042 obj_retry.go:458] Detected object konflux-ui/proxy-86995756bd-q2bm6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:02.096623 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-86995756bd-q2bm6 I0331 04:08:02.096650 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-86995756bd-q2bm6 I0331 04:08:04.539872 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=159 I0331 04:08:10.611314 4042 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:10.611940 4042 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:10.630682 4042 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:10.630701 4042 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:10.637766 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0331 04:08:10.637786 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0331 04:08:37.539246 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0331 04:08:37.548020 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=518 I0331 04:08:37.891809 4042 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr cf1f13ed26575336c254197a1b7d9a977a035625a5d2cccff4241148b9df852d network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr cf1f13ed26575336c254197a1b7d9a977a035625a5d2cccff4241148b9df852d network default NAD default] I0331 04:08:37.979695 4042 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr cf1f13ed26575336c254197a1b7d9a977a035625a5d2cccff4241148b9df852d network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr cf1f13ed26575336c254197a1b7d9a977a035625a5d2cccff4241148b9df852d network default NAD default], result "{}", err I0331 04:08:38.044191 4042 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:38.044220 4042 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr I0331 04:08:38.045757 4042 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr, ips: 10.128.0.28 I0331 04:08:38.045785 4042 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:38.481276 4042 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:38.481301 4042 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:38.489081 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr I0331 04:08:38.489105 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-nk4zr I0331 04:08:52.671921 4042 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-n6vjd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:52.672437 4042 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-n6vjd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:52.990128 4042 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-n6vjd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:52.990150 4042 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-n6vjd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:52.998145 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-n6vjd I0331 04:08:52.998175 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-n6vjd I0331 04:09:18.637710 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0331 04:09:45.628223 4042 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0331 04:10:03.974524 4042 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-khgdx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:03.975063 4042 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-khgdx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:04.856849 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-khgdx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:04.856890 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-khgdx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:18.655644 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0331 04:10:29.669196 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0331 04:10:49.539525 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=42 I0331 04:11:08.633561 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0331 04:11:40.627942 4042 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0331 04:12:11.543610 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=57 I0331 04:12:22.668605 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0331 04:13:21.661944 4042 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0331 04:13:40.543241 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=6 I0331 04:14:16.649936 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0331 04:15:00.644272 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0331 04:15:29.550281 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=34 I0331 04:15:45.630433 4042 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0331 04:15:48.057117 4042 obj_retry.go:458] Detected object tsf-tpa/create-db-gkhp4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:48.057160 4042 obj_retry.go:458] Detected object tsf-tpa/create-db-gkhp4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:48.067651 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-gkhp4 I0331 04:15:48.067682 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-gkhp4 I0331 04:15:48.658601 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0331 04:15:52.034947 4042 obj_retry.go:458] Detected object tsf-tpa/migrate-db-sdrs7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:52.034990 4042 obj_retry.go:458] Detected object tsf-tpa/migrate-db-sdrs7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:52.041917 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-sdrs7 I0331 04:15:52.041941 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-sdrs7 I0331 04:15:55.038866 4042 obj_retry.go:458] Detected object tsf-tpa/create-importers-8bsnv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:55.038892 4042 obj_retry.go:458] Detected object tsf-tpa/create-importers-8bsnv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:55.044978 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-8bsnv I0331 04:15:55.045004 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-8bsnv I0331 04:16:08.542206 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0331 04:16:27.640929 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0331 04:16:49.672287 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0331 04:17:39.543708 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=43 I0331 04:18:02.542924 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8 I0331 04:19:42.664370 4042 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0331 04:19:45.636084 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0331 04:20:04.558364 4042 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cn6f4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:20:04.558974 4042 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cn6f4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:20:05.429158 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cn6f4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:05.429215 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-cn6f4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:05.464056 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:05.464115 4042 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:05.467807 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk I0331 04:20:05.467833 4042 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29582150-b9wzk I0331 04:20:44.545762 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0331 04:20:55.552569 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=23 I0331 04:20:58.652345 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0331 04:21:04.630667 4042 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0331 04:21:35.634010 4042 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0331 04:21:41.670973 4042 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0331 04:22:10.550247 4042 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=10