+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-29.ec2.internal ++ [[ -n ip-10-0-0-29.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-29.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0331 03:58:41.244473716 - disable conntrack on geneve port' I0331 03:58:41.244473716 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0331 03:58:41.306613064 - starting ovnkube-node + echo 'I0331 03:58:41.306613064 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-29.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-29.ec2.internal --init-node ip-10-0-0-29.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-29.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0331 03:58:41.359687 3934 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0331 03:58:41.359728 3934 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-21ecbe3741.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0331 03:58:41.361255 3934 kube.go:419] Waiting for certificate I0331 03:58:41.361305 3934 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0331 03:58:41.361341 3934 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0331 03:58:41.362196 3934 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0331 03:58:41.392234 3934 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:58:41.392253 3934 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:58:41.395771 3934 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:58:41.395857 3934 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-ggd5r" I0331 03:58:52.622459 3934 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-ggd5r" I0331 03:58:52.622592 3934 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0331 03:58:53.361492 3934 kube.go:426] Certificate found I0331 03:58:53.362181 3934 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0331 03:58:53.363263 3934 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0331 03:58:53.436289 3934 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0331 03:58:53.447480 3934 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0331 03:58:53.447566 3934 factory.go:531] Starting watch factory I0331 03:58:53.447751 3934 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.447772 3934 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.447867 3934 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.447884 3934 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.447913 3934 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0331 03:58:53.447990 3934 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.448025 3934 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.448227 3934 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.448314 3934 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.448644 3934 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.448675 3934 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.451336 3934 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.451461 3934 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.455473 3934 controller_manager.go:368] Starting the ovnkube controller I0331 03:58:53.455482 3934 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-29.ec2.internal I0331 03:58:53.455523 3934 controller_manager.go:393] NBDB zone sync took: 33.848µs I0331 03:58:53.455531 3934 factory.go:531] Starting watch factory I0331 03:58:53.478185 3934 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.480072 3934 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.480991 3934 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.481210 3934 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.481305 3934 ovn_northd.go:121] Registering OVN northd metrics I0331 03:58:53.481445 3934 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0331 03:58:53.482641 3934 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.485545 3934 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0331 03:58:53.485617 3934 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0331 03:58:53.493061 3934 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0331 03:58:53.501661 3934 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0331 03:58:53.552782 3934 factory.go:1890] *v1.Service informer cache synced successfully I0331 03:58:53.552798 3934 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0331 03:58:53.552801 3934 factory.go:1890] *v1.Pod informer cache synced successfully I0331 03:58:53.552805 3934 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0331 03:58:53.552808 3934 factory.go:1890] *v1.Namespace informer cache synced successfully I0331 03:58:53.552811 3934 factory.go:1890] *v1.Node informer cache synced successfully I0331 03:58:53.552872 3934 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:58:53.552885 3934 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:58:53.552917 3934 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:58:53.552937 3934 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:58:53.556344 3934 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:58:53.556588 3934 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0331 03:58:53.558591 3934 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0331 03:58:53.558603 3934 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0331 03:58:53.558648 3934 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.558659 3934 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.561610 3934 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0331 03:58:53.561621 3934 factory.go:1890] *v1.Namespace informer cache synced successfully I0331 03:58:53.561625 3934 factory.go:1890] *v1.Node informer cache synced successfully I0331 03:58:53.561630 3934 factory.go:1890] *v1.Service informer cache synced successfully I0331 03:58:53.561633 3934 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0331 03:58:53.561635 3934 factory.go:1890] *v1.Pod informer cache synced successfully I0331 03:58:53.561732 3934 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.563783 3934 factory.go:1890] *v1.EgressIP informer cache synced successfully I0331 03:58:53.563833 3934 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.563843 3934 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.566658 3934 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0331 03:58:53.566667 3934 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0331 03:58:53.566868 3934 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.569730 3934 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0331 03:58:53.569783 3934 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.569793 3934 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.572228 3934 factory.go:1890] *v1.EgressIP informer cache synced successfully I0331 03:58:53.572602 3934 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.575557 3934 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0331 03:58:53.575604 3934 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.575616 3934 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.578011 3934 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0331 03:58:53.578538 3934 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.581518 3934 factory.go:1890] *v1.EgressService informer cache synced successfully I0331 03:58:53.581574 3934 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.581587 3934 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.583727 3934 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0331 03:58:53.584859 3934 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.586837 3934 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0331 03:58:53.586992 3934 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:58:53.587004 3934 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:58:53.588790 3934 factory.go:1890] *v1.EgressService informer cache synced successfully I0331 03:58:53.590042 3934 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0331 03:58:53.592027 3934 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0331 03:58:53.592086 3934 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.592102 3934 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.592147 3934 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.592164 3934 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.593907 3934 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0331 03:58:53.595985 3934 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.596260 3934 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0331 03:58:53.597181 3934 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0331 03:58:53.597191 3934 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0331 03:58:53.597197 3934 factory.go:636] Watch Factory start up complete, took: 149.638787ms I0331 03:58:53.597257 3934 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0331 03:58:53.599140 3934 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0331 03:58:53.605064 3934 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0331 03:58:53.605072 3934 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0331 03:58:53.605078 3934 factory.go:636] Watch Factory start up complete, took: 149.545586ms I0331 03:58:53.605085 3934 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-0-29.ec2.internal" zone I0331 03:58:53.650872 3934 default_node_network_controller.go:709] Initializing the default node network controller I0331 03:58:53.661750 3934 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-01 03:53:52 +0000 UTC" deadline="2026-03-31 23:59:42.862345399 +0000 UTC" I0331 03:58:53.661788 3934 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h0m49.200563886s" I0331 03:58:53.723209 3934 udn_isolation.go:101] Starting UDN host isolation manager I0331 03:58:53.729457 3934 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0331 03:58:53.783046 3934 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0331 03:58:53.783219 3934 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0331 03:58:53.783235 3934 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0331 03:58:53.814935 3934 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0331 03:58:53.814990 3934 default_node_network_controller.go:808] Node ip-10-0-0-29.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0331 03:58:53.820212 3934 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.29"] k8s.ovn.org/zone-name:ip-10-0-0-29.ec2.internal] on node ip-10-0-0-29.ec2.internal I0331 03:58:53.840857 3934 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0331 03:58:53.845903 3934 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0331 03:58:53.846004 3934 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0331 03:58:53.846142 3934 gateway_init.go:263] Preparing Gateway I0331 03:58:53.846154 3934 gateway_shared_intf.go:1658] Creating new gateway I0331 03:58:53.855274 3934 controller_manager.go:427] Waiting for node in zone sync took: 250.175747ms I0331 03:58:53.860538 3934 controller_manager.go:608] Switching to transit router for layer2 networks I0331 03:58:53.860553 3934 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-29.ec2.internal I0331 03:58:53.876701 3934 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0331 03:58:53.882049 3934 controller_manager.go:288] SCTP support detected in OVN I0331 03:58:53.882629 3934 services_controller.go:69] Creating services controller for network=default I0331 03:58:53.882735 3934 udn_enabled_svc.go:80] Waiting for service informer to sync I0331 03:58:53.882770 3934 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0331 03:58:53.882834 3934 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0331 03:58:53.882847 3934 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0331 03:58:53.882857 3934 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0331 03:58:53.883019 3934 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0331 03:58:53.883065 3934 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0331 03:58:53.883093 3934 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 45.266µs I0331 03:58:53.883118 3934 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 40.518µs I0331 03:58:53.883136 3934 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 67.576µs I0331 03:58:53.883144 3934 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0331 03:58:53.883158 3934 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0331 03:58:53.883175 3934 default_network_controller.go:321] Starting the default network controller I0331 03:58:53.883220 3934 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 12.608µs I0331 03:58:53.883249 3934 udn_enabled_svc.go:95] Performing full resync I0331 03:58:53.883554 3934 udn_enabled_svc.go:99] Waiting for handler to sync I0331 03:58:53.883564 3934 udn_enabled_svc.go:104] Starting worker I0331 03:58:53.891588 3934 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0331 03:58:53.891657 3934 repair.go:33] Syncing exgw routes took 53.287µs I0331 03:58:53.891673 3934 default_network_controller.go:407] Starting all the Watchers... I0331 03:58:53.891919 3934 namespace.go:98] [openshift-cluster-version] adding namespace I0331 03:58:53.891931 3934 namespace.go:98] [openshift-console] adding namespace I0331 03:58:53.891933 3934 namespace.go:98] [openshift-image-registry] adding namespace I0331 03:58:53.891951 3934 namespace.go:98] [openshift-config-operator] adding namespace I0331 03:58:53.891952 3934 namespace.go:98] [openshift-config] adding namespace I0331 03:58:53.891960 3934 namespace.go:98] [openshift-config-managed] adding namespace I0331 03:58:53.891972 3934 namespace.go:98] [openshift-machine-config-operator] adding namespace I0331 03:58:53.891971 3934 namespace.go:98] [openshift-route-controller-manager] adding namespace I0331 03:58:53.891983 3934 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0331 03:58:53.891988 3934 namespace.go:98] [open-cluster-management-2pcf6lkp4abd3rm3r9e30v12cg53upbr] adding namespace I0331 03:58:53.891992 3934 namespace.go:98] [openshift-apiserver-operator] adding namespace I0331 03:58:53.891997 3934 namespace.go:98] [openshift-backplane-mobb] adding namespace I0331 03:58:53.891999 3934 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0331 03:58:53.892002 3934 namespace.go:98] [openshift-machine-api] adding namespace I0331 03:58:53.892052 3934 namespace.go:98] [openshift-network-diagnostics] adding namespace I0331 03:58:53.892516 3934 namespace.go:102] [openshift-cluster-version] adding namespace took 583.523µs I0331 03:58:53.892533 3934 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0331 03:58:53.893110 3934 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 571.417µs I0331 03:58:53.893125 3934 namespace.go:98] [openshift-operators] adding namespace I0331 03:58:53.893754 3934 namespace.go:102] [openshift-operators] adding namespace took 622.704µs I0331 03:58:53.893767 3934 namespace.go:98] [openshift-ingress-canary] adding namespace I0331 03:58:53.894476 3934 namespace.go:102] [openshift-console] adding namespace took 2.536556ms I0331 03:58:53.894486 3934 namespace.go:98] [openshift-multus] adding namespace I0331 03:58:53.895104 3934 namespace.go:102] [openshift-image-registry] adding namespace took 3.157629ms I0331 03:58:53.895124 3934 namespace.go:98] [openshift-ingress-operator] adding namespace I0331 03:58:53.895732 3934 namespace.go:102] [openshift-config-operator] adding namespace took 3.773652ms I0331 03:58:53.895746 3934 namespace.go:98] [openshift-insights] adding namespace I0331 03:58:53.896357 3934 namespace.go:102] [openshift-config-managed] adding namespace took 4.390386ms I0331 03:58:53.896375 3934 namespace.go:98] [openshift-kube-apiserver] adding namespace I0331 03:58:53.896906 3934 namespace.go:102] [openshift-config] adding namespace took 4.943169ms I0331 03:58:53.896918 3934 namespace.go:98] [openshift-backplane-csm] adding namespace I0331 03:58:53.897381 3934 namespace.go:102] [openshift-machine-config-operator] adding namespace took 5.403329ms I0331 03:58:53.897395 3934 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0331 03:58:53.898028 3934 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.044409ms I0331 03:58:53.898045 3934 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0331 03:58:53.898509 3934 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.5181ms I0331 03:58:53.898526 3934 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0331 03:58:53.899074 3934 namespace.go:102] [open-cluster-management-2pcf6lkp4abd3rm3r9e30v12cg53upbr] adding namespace took 7.078197ms I0331 03:58:53.899089 3934 namespace.go:98] [openshift-monitoring] adding namespace I0331 03:58:53.899839 3934 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.840935ms I0331 03:58:53.899849 3934 namespace.go:98] [openshift-marketplace] adding namespace I0331 03:58:53.900385 3934 namespace.go:102] [openshift-backplane-mobb] adding namespace took 8.38453ms I0331 03:58:53.900393 3934 namespace.go:98] [openshift-dns] adding namespace I0331 03:58:53.900929 3934 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 8.9231ms I0331 03:58:53.900949 3934 namespace.go:98] [openshift-backplane] adding namespace I0331 03:58:53.901516 3934 namespace.go:102] [openshift-machine-api] adding namespace took 9.506842ms W0331 03:58:53.901518 3934 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-6b6dn: pod openshift-network-diagnostics/network-check-target-6b6dn: no pod IPs found I0331 03:58:53.901533 3934 namespace.go:98] [openshift-authentication] adding namespace W0331 03:58:53.901538 3934 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-wp4rx: pod openshift-network-diagnostics/network-check-target-wp4rx: no pod IPs found I0331 03:58:53.902212 3934 namespace.go:102] [openshift-network-diagnostics] adding namespace took 10.152099ms I0331 03:58:53.902222 3934 namespace.go:98] [openshift-logging] adding namespace W0331 03:58:53.902741 3934 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-z6hdt: pod openshift-multus/network-metrics-daemon-z6hdt: no pod IPs found W0331 03:58:53.902753 3934 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-hjsvb: pod openshift-multus/network-metrics-daemon-hjsvb: no pod IPs found I0331 03:58:53.902763 3934 namespace.go:102] [openshift-ingress-canary] adding namespace took 8.988823ms I0331 03:58:53.902776 3934 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0331 03:58:53.903289 3934 namespace.go:102] [openshift-multus] adding namespace took 8.798756ms I0331 03:58:53.903300 3934 namespace.go:98] [openshift-host-network] adding namespace I0331 03:58:53.903744 3934 namespace.go:102] [openshift-ingress-operator] adding namespace took 8.61597ms I0331 03:58:53.903755 3934 namespace.go:98] [openshift-network-console] adding namespace I0331 03:58:53.904298 3934 namespace.go:102] [openshift-insights] adding namespace took 8.54833ms I0331 03:58:53.904308 3934 namespace.go:98] [kube-system] adding namespace I0331 03:58:53.904792 3934 namespace.go:102] [openshift-kube-apiserver] adding namespace took 8.407474ms I0331 03:58:53.904813 3934 namespace.go:98] [openshift-node] adding namespace I0331 03:58:53.905333 3934 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.406864ms I0331 03:58:53.905355 3934 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0331 03:58:53.905921 3934 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 8.519671ms I0331 03:58:53.905937 3934 namespace.go:98] [openshift] adding namespace I0331 03:58:53.906467 3934 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 8.416372ms I0331 03:58:53.906481 3934 namespace.go:98] [openshift-apiserver] adding namespace I0331 03:58:53.906915 3934 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 8.383156ms I0331 03:58:53.906936 3934 namespace.go:98] [openshift-must-gather-operator] adding namespace I0331 03:58:53.907530 3934 namespace.go:102] [openshift-monitoring] adding namespace took 8.432024ms I0331 03:58:53.907545 3934 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0331 03:58:53.908088 3934 namespace.go:102] [openshift-marketplace] adding namespace took 8.232974ms I0331 03:58:53.908107 3934 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0331 03:58:53.908621 3934 namespace.go:102] [openshift-dns] adding namespace took 8.219874ms I0331 03:58:53.908639 3934 namespace.go:98] [openshift-console-operator] adding namespace I0331 03:58:53.909153 3934 namespace.go:102] [openshift-backplane] adding namespace took 8.19934ms I0331 03:58:53.909164 3934 namespace.go:98] [openshift-kube-scheduler] adding namespace I0331 03:58:53.909655 3934 namespace.go:102] [openshift-authentication] adding namespace took 8.117339ms I0331 03:58:53.909666 3934 namespace.go:98] [openshift-operators-redhat] adding namespace I0331 03:58:53.910226 3934 namespace.go:102] [openshift-logging] adding namespace took 8.000095ms I0331 03:58:53.910236 3934 namespace.go:98] [kube-public] adding namespace I0331 03:58:53.910794 3934 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 8.01202ms I0331 03:58:53.911296 3934 namespace.go:102] [openshift-host-network] adding namespace took 7.989297ms I0331 03:58:53.911312 3934 namespace.go:98] [openshift-dns-operator] adding namespace W0331 03:58:53.911803 3934 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-dz4n2: pod kube-system/global-pull-secret-syncer-dz4n2: no pod IPs found I0331 03:58:53.911845 3934 namespace.go:102] [openshift-network-console] adding namespace took 8.085595ms I0331 03:58:53.911855 3934 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0331 03:58:53.912356 3934 namespace.go:102] [kube-system] adding namespace took 8.043788ms I0331 03:58:53.912366 3934 namespace.go:98] [openshift-service-ca-operator] adding namespace I0331 03:58:53.912802 3934 namespace.go:102] [openshift-node] adding namespace took 7.981634ms I0331 03:58:53.912816 3934 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0331 03:58:53.913268 3934 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.906783ms I0331 03:58:53.913282 3934 namespace.go:98] [default] adding namespace I0331 03:58:53.913722 3934 namespace.go:102] [openshift] adding namespace took 7.777613ms I0331 03:58:53.913742 3934 namespace.go:98] [openshift-backplane-cse] adding namespace I0331 03:58:53.914196 3934 namespace.go:102] [openshift-apiserver] adding namespace took 7.70988ms I0331 03:58:53.914210 3934 namespace.go:98] [dedicated-admin] adding namespace I0331 03:58:53.914609 3934 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.666978ms I0331 03:58:53.914619 3934 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0331 03:58:53.915012 3934 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.461599ms I0331 03:58:53.915026 3934 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0331 03:58:53.915403 3934 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.290939ms I0331 03:58:53.915415 3934 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0331 03:58:53.915791 3934 namespace.go:102] [openshift-console-operator] adding namespace took 7.146317ms I0331 03:58:53.915802 3934 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0331 03:58:53.916195 3934 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.027103ms I0331 03:58:53.916207 3934 namespace.go:98] [openshift-backplane-tam] adding namespace I0331 03:58:53.916582 3934 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.911527ms I0331 03:58:53.916599 3934 namespace.go:98] [openshift-service-ca] adding namespace I0331 03:58:53.916972 3934 namespace.go:102] [kube-public] adding namespace took 6.731527ms I0331 03:58:53.916983 3934 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0331 03:58:53.917357 3934 namespace.go:102] [openshift-dns-operator] adding namespace took 6.040926ms I0331 03:58:53.917368 3934 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0331 03:58:53.917727 3934 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 5.868086ms I0331 03:58:53.917737 3934 namespace.go:98] [openshift-customer-monitoring] adding namespace I0331 03:58:53.918182 3934 namespace.go:102] [openshift-service-ca-operator] adding namespace took 5.811814ms I0331 03:58:53.918193 3934 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0331 03:58:53.918558 3934 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 5.737484ms I0331 03:58:53.918569 3934 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0331 03:58:53.918915 3934 namespace.go:102] [default] adding namespace took 5.629048ms I0331 03:58:53.918924 3934 namespace.go:98] [openshift-backplane-srep] adding namespace I0331 03:58:53.919339 3934 namespace.go:102] [openshift-backplane-cse] adding namespace took 5.591669ms I0331 03:58:53.919352 3934 namespace.go:98] [openshift-ingress] adding namespace I0331 03:58:53.919695 3934 namespace.go:102] [dedicated-admin] adding namespace took 5.479765ms I0331 03:58:53.919706 3934 namespace.go:98] [openshift-package-operator] adding namespace I0331 03:58:53.920114 3934 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 5.490833ms I0331 03:58:53.920126 3934 namespace.go:98] [kube-node-lease] adding namespace I0331 03:58:53.920521 3934 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 5.489087ms I0331 03:58:53.920533 3934 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0331 03:58:53.920916 3934 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 5.494287ms I0331 03:58:53.920927 3934 namespace.go:98] [openshift-network-node-identity] adding namespace I0331 03:58:53.921348 3934 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 5.541008ms I0331 03:58:53.921360 3934 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0331 03:58:53.921700 3934 namespace.go:102] [openshift-backplane-tam] adding namespace took 5.48807ms I0331 03:58:53.921713 3934 namespace.go:98] [openshift-controller-manager] adding namespace I0331 03:58:53.922451 3934 namespace.go:102] [openshift-service-ca] adding namespace took 5.846369ms I0331 03:58:53.922463 3934 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0331 03:58:53.922814 3934 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 5.826532ms I0331 03:58:53.922824 3934 namespace.go:98] [openshift-infra] adding namespace I0331 03:58:53.923222 3934 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 5.849914ms I0331 03:58:53.923609 3934 namespace.go:102] [openshift-customer-monitoring] adding namespace took 5.867776ms I0331 03:58:53.923992 3934 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 5.795101ms I0331 03:58:53.924359 3934 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 5.785744ms I0331 03:58:53.924692 3934 namespace.go:102] [openshift-backplane-srep] adding namespace took 5.763738ms I0331 03:58:53.924703 3934 namespace.go:98] [openshift-authentication-operator] adding namespace I0331 03:58:53.925164 3934 namespace.go:102] [openshift-ingress] adding namespace took 5.806777ms I0331 03:58:53.925177 3934 namespace.go:98] [openshift-console-user-settings] adding namespace I0331 03:58:53.925533 3934 namespace.go:102] [openshift-package-operator] adding namespace took 5.822631ms I0331 03:58:53.925851 3934 namespace.go:102] [kube-node-lease] adding namespace took 5.721316ms I0331 03:58:53.925862 3934 namespace.go:98] [openshift-network-operator] adding namespace I0331 03:58:53.926332 3934 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 5.794512ms I0331 03:58:53.926698 3934 namespace.go:102] [openshift-network-node-identity] adding namespace took 5.766921ms I0331 03:58:53.926709 3934 namespace.go:98] [openshift-etcd] adding namespace I0331 03:58:53.927081 3934 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 5.716083ms I0331 03:58:53.927491 3934 namespace.go:102] [openshift-controller-manager] adding namespace took 5.772852ms I0331 03:58:53.927845 3934 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 5.377316ms I0331 03:58:53.927856 3934 namespace.go:98] [openshift-backplane-cee] adding namespace I0331 03:58:53.928315 3934 namespace.go:102] [openshift-infra] adding namespace took 5.485414ms I0331 03:58:53.928330 3934 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0331 03:58:53.928708 3934 namespace.go:102] [openshift-authentication-operator] adding namespace took 4.000697ms I0331 03:58:53.929063 3934 namespace.go:102] [openshift-console-user-settings] adding namespace took 3.881436ms I0331 03:58:53.929498 3934 namespace.go:102] [openshift-network-operator] adding namespace took 3.631234ms I0331 03:58:53.929821 3934 namespace.go:102] [openshift-etcd] adding namespace took 3.107398ms I0331 03:58:53.930275 3934 namespace.go:102] [openshift-backplane-cee] adding namespace took 2.414681ms I0331 03:58:53.930619 3934 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 2.285124ms I0331 03:58:53.930975 3934 zone_ic_handler.go:186] Time taken to create transit switch: 269.738µs I0331 03:58:53.931040 3934 master.go:532] Adding or Updating local node "ip-10-0-0-29.ec2.internal" for network "default" I0331 03:58:53.931115 3934 obj_retry.go:544] Failed to create *v1.Node ip-10-0-2-235.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-235.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-235.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-235.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-235.ec2.internal I0331 03:58:53.931565 3934 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-50.ec2.internal for the network default I0331 03:58:53.933269 3934 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-50.ec2.internal" on network "default" took: 1.71418ms I0331 03:58:53.935324 3934 default_network_controller.go:806] Node add failed for ip-10-0-0-29.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-29.ec2.internal, error getting gateway config for node ip-10-0-0-29.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-29.ec2.internal", failed to update chassis to local for local node ip-10-0-0-29.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-29.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-29.ec2.internal] E0331 03:58:53.935380 3934 obj_retry.go:541] Failed to create *v1.Node ip-10-0-0-29.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-29.ec2.internal, error getting gateway config for node ip-10-0-0-29.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-29.ec2.internal", failed to update chassis to local for local node ip-10-0-0-29.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-29.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-29.ec2.internal] I0331 03:58:53.935430 3934 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0331 03:58:53.935456 3934 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0331 03:58:53.935447 3934 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-29.ec2.internal", UID:"1ac4740a-7f40-40f6-9ead-618256a1df3f", APIVersion:"v1", ResourceVersion:"10211", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-29.ec2.internal, error getting gateway config for node ip-10-0-0-29.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-29.ec2.internal", failed to update chassis to local for local node ip-10-0-0-29.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-29.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-29.ec2.internal] I0331 03:58:53.935470 3934 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0331 03:58:53.935472 3934 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-29.ec2.internal in network "default" I0331 03:58:53.935505 3934 node_tracker.go:235] Node ip-10-0-0-29.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-29.ec2.internal" I0331 03:58:53.935519 3934 node_tracker.go:169] Node ip-10-0-0-29.ec2.internal switch + router changed, syncing services in network "default" I0331 03:58:53.935527 3934 services_controller.go:574] Full service sync requested for network=default I0331 03:58:53.935532 3934 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-50.ec2.internal in network "default" I0331 03:58:53.935570 3934 node_tracker.go:169] Node ip-10-0-1-50.ec2.internal switch + router changed, syncing services in network "default" I0331 03:58:53.935577 3934 services_controller.go:574] Full service sync requested for network=default I0331 03:58:53.935584 3934 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-235.ec2.internal in network "default" I0331 03:58:53.935595 3934 node_tracker.go:235] Node ip-10-0-2-235.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-235.ec2.internal" I0331 03:58:53.935600 3934 node_tracker.go:169] Node ip-10-0-2-235.ec2.internal switch + router changed, syncing services in network "default" I0331 03:58:53.935606 3934 services_controller.go:574] Full service sync requested for network=default I0331 03:58:53.964322 3934 iptables.go:108] Creating table: filter chain: FORWARD I0331 03:58:53.970853 3934 iptables.go:108] Creating table: filter chain: OUTPUT I0331 03:58:53.977626 3934 gateway_shared_intf.go:1787] Gateway Creation Complete I0331 03:58:53.977641 3934 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-29.ec2.internal","mac-address":"12:c7:7a:4d:c1:cd","ip-addresses":["10.0.0.29/24"],"ip-address":"10.0.0.29/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:bfb9571c-b847-4d90-9834-6f5292f11107 k8s.ovn.org/node-encap-ips:["10.0.0.29"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.29/24"} k8s.ovn.org/zone-name:ip-10-0-0-29.ec2.internal] on node ip-10-0-0-29.ec2.internal I0331 03:58:53.995394 3934 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-29.ec2.internal in network "default" I0331 03:58:53.995450 3934 node_tracker.go:169] Node ip-10-0-0-29.ec2.internal switch + router changed, syncing services in network "default" I0331 03:58:53.995447 3934 master.go:532] Adding or Updating local node "ip-10-0-0-29.ec2.internal" for network "default" I0331 03:58:53.995464 3934 services_controller.go:574] Full service sync requested for network=default I0331 03:58:53.998839 3934 default_node_network_controller.go:518] Node connection status = connected W0331 03:58:54.000184 3934 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-6b6dn: pod openshift-network-diagnostics/network-check-target-6b6dn: no pod IPs found W0331 03:58:54.000204 3934 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-dz4n2: pod kube-system/global-pull-secret-syncer-dz4n2: no pod IPs found W0331 03:58:54.000211 3934 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-hjsvb: pod openshift-multus/network-metrics-daemon-hjsvb: no pod IPs found I0331 03:58:54.001355 3934 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-0-29.ec2.internal for the network default I0331 03:58:54.035548 3934 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0331 03:58:54.035568 3934 services_controller.go:208] Setting up event handlers for services for network=default I0331 03:58:54.035643 3934 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0331 03:58:54.035684 3934 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0331 03:58:54.035698 3934 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0331 03:58:54.136185 3934 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0331 03:58:54.136206 3934 repair.go:60] Starting repairing loop for services I0331 03:58:54.136285 3934 repair.go:131] Deleted 0 stale service LBs I0331 03:58:54.136304 3934 repair.go:137] Deleted 0 stale Chassis Template Vars I0331 03:58:54.136320 3934 repair.go:62] Finished repairing loop for services: 116.106µs I0331 03:58:54.136344 3934 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0331 03:58:54.136354 3934 services_controller.go:254] Starting workers for network=default I0331 03:58:54.136843 3934 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-dz4n2] creating logical port kube-system_global-pull-secret-syncer-dz4n2 for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:54.136933 3934 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-wp4rx, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-wp4rx: suppressed error logged: pod openshift-network-diagnostics/network-check-target-wp4rx: no pod IPs found I0331 03:58:54.136995 3934 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-6b6dn] creating logical port openshift-network-diagnostics_network-check-target-6b6dn for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:54.137041 3934 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-dz4n2 I0331 03:58:54.137085 3934 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-6b6dn I0331 03:58:54.137170 3934 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-z6hdt, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-z6hdt: suppressed error logged: pod openshift-multus/network-metrics-daemon-z6hdt: no pod IPs found I0331 03:58:54.137308 3934 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-hjsvb] creating logical port openshift-multus_network-metrics-daemon-hjsvb for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:54.137389 3934 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-hjsvb W0331 03:58:54.137535 3934 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:58:54.140200 3934 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:58:54.142350 3934 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:54.142365 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:54.142369 3934 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0331 03:58:54.153730 3934 pod.go:62] [kube-system/global-pull-secret-syncer-dz4n2] pod update took 16.764404ms I0331 03:58:54.153749 3934 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-dz4n2] addLogicalPort annotation time took 16.788451ms I0331 03:58:54.154681 3934 pods.go:271] [kube-system/global-pull-secret-syncer-dz4n2] addLogicalPort took 17.852251ms, libovsdb time 694.899µs I0331 03:58:54.154717 3934 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-k8rxr of type *v1.Pod in terminal state (e.g. completed) during add event: will remove it I0331 03:58:54.158872 3934 pod.go:62] [openshift-network-diagnostics/network-check-target-6b6dn] pod update took 21.830289ms I0331 03:58:54.158909 3934 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-6b6dn] addLogicalPort annotation time took 21.850793ms I0331 03:58:54.159366 3934 pod.go:62] [openshift-multus/network-metrics-daemon-hjsvb] pod update took 22.004231ms I0331 03:58:54.159549 3934 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-hjsvb] addLogicalPort annotation time took 22.182935ms I0331 03:58:54.159958 3934 pods.go:271] [openshift-network-diagnostics/network-check-target-6b6dn] addLogicalPort took 22.968558ms, libovsdb time 826.095µs I0331 03:58:54.160531 3934 pods.go:271] [openshift-multus/network-metrics-daemon-hjsvb] addLogicalPort took 23.232016ms, libovsdb time 720.105µs I0331 03:58:54.161082 3934 admin_network_policy_controller.go:227] Starting controller default-network-controller I0331 03:58:54.161101 3934 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0331 03:58:54.161112 3934 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0331 03:58:54.161116 3934 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0331 03:58:54.161350 3934 repair.go:29] Repairing admin network policies took 227.749µs I0331 03:58:54.161483 3934 repair.go:92] Repairing baseline admin network policies took 123.97µs I0331 03:58:54.162076 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0331 03:58:54.162105 3934 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0331 03:58:54.164341 3934 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0331 03:58:54.164362 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0331 03:58:54.164372 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0331 03:58:54.165998 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0331 03:58:54.166019 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0331 03:58:54.166029 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0331 03:58:54.167632 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0331 03:58:54.167655 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0331 03:58:54.168694 3934 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0331 03:58:54.168708 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0331 03:58:54.169625 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0331 03:58:54.169643 3934 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0331 03:58:54.169651 3934 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0331 03:58:54.171463 3934 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0331 03:58:54.171481 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0331 03:58:54.171491 3934 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0331 03:58:54.171946 3934 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0331 03:58:54.171959 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0331 03:58:54.171977 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0331 03:58:54.172549 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0331 03:58:54.172577 3934 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0331 03:58:54.172585 3934 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0331 03:58:54.174545 3934 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0331 03:58:54.174568 3934 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0331 03:58:54.174579 3934 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0331 03:58:54.176649 3934 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0331 03:58:54.176675 3934 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0331 03:58:54.176693 3934 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0331 03:58:54.179284 3934 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0331 03:58:54.179308 3934 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0331 03:58:54.179321 3934 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0331 03:58:54.180309 3934 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0331 03:58:54.180328 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0331 03:58:54.180337 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0331 03:58:54.180872 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0331 03:58:54.180909 3934 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0331 03:58:54.180931 3934 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0331 03:58:54.182066 3934 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0331 03:58:54.182083 3934 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0331 03:58:54.182092 3934 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0331 03:58:54.182984 3934 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0331 03:58:54.183001 3934 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0331 03:58:54.183009 3934 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0331 03:58:54.183352 3934 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0331 03:58:54.183375 3934 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0331 03:58:54.183392 3934 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0331 03:58:54.184624 3934 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0331 03:58:54.184641 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0331 03:58:54.184647 3934 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0331 03:58:54.184954 3934 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0331 03:58:54.184969 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0331 03:58:54.184977 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0331 03:58:54.185464 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0331 03:58:54.185488 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0331 03:58:54.185501 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0331 03:58:54.186787 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0331 03:58:54.186808 3934 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0331 03:58:54.186819 3934 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0331 03:58:54.187387 3934 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0331 03:58:54.187406 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0331 03:58:54.187416 3934 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0331 03:58:54.187849 3934 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0331 03:58:54.187867 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0331 03:58:54.187878 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0331 03:58:54.188372 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0331 03:58:54.188390 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0331 03:58:54.188396 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0331 03:58:54.188681 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0331 03:58:54.188697 3934 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0331 03:58:54.188706 3934 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0331 03:58:54.189675 3934 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0331 03:58:54.189693 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0331 03:58:54.189714 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0331 03:58:54.190409 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0331 03:58:54.190434 3934 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0331 03:58:54.190444 3934 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0331 03:58:54.190951 3934 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0331 03:58:54.190967 3934 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0331 03:58:54.190973 3934 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0331 03:58:54.191230 3934 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0331 03:58:54.191244 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0331 03:58:54.191250 3934 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0331 03:58:54.191736 3934 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0331 03:58:54.191756 3934 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0331 03:58:54.191768 3934 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0331 03:58:54.192714 3934 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0331 03:58:54.192955 3934 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-k8rxr of type *factory.egressIPPod in terminal state (e.g. completed) during add event: will remove it I0331 03:58:54.198262 3934 egressfirewall.go:319] Starting EgressFirewall controller I0331 03:58:54.198292 3934 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0331 03:58:54.198310 3934 controller.go:133] Adding controller egress-firewall-controller event handlers I0331 03:58:54.198324 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0331 03:58:54.198329 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0331 03:58:54.198342 3934 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0331 03:58:54.198347 3934 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0331 03:58:54.198591 3934 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0331 03:58:54.198616 3934 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0331 03:58:54.198627 3934 egressqos.go:193] Setting up event handlers for EgressQoS I0331 03:58:54.198644 3934 egressfirewall.go:628] Syncing node "ip-10-0-0-29.ec2.internal" for egress firewall I0331 03:58:54.198658 3934 egressfirewall.go:628] Syncing node "ip-10-0-1-50.ec2.internal" for egress firewall I0331 03:58:54.198662 3934 egressfirewall.go:628] Syncing node "ip-10-0-2-235.ec2.internal" for egress firewall I0331 03:58:54.198798 3934 egressqos.go:245] Starting EgressQoS Controller I0331 03:58:54.198814 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0331 03:58:54.198822 3934 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0331 03:58:54.198827 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0331 03:58:54.198831 3934 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0331 03:58:54.198836 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0331 03:58:54.198840 3934 shared_informer.go:356] "Caches are synced" controller="egressqos" I0331 03:58:54.198844 3934 egressqos.go:259] Repairing EgressQoSes I0331 03:58:54.198849 3934 egressqos.go:399] Starting repairing loop for egressqos I0331 03:58:54.198945 3934 egressqos.go:401] Finished repairing loop for egressqos: 96.106µs I0331 03:58:54.198960 3934 egressservice_zone.go:132] Setting up event handlers for Egress Services I0331 03:58:54.199000 3934 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-29.ec2.internal I0331 03:58:54.199037 3934 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-29.ec2.internal : 38.733µs I0331 03:58:54.199096 3934 egressservice_zone.go:210] Starting Egress Services Controller I0331 03:58:54.199105 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0331 03:58:54.199110 3934 shared_informer.go:356] "Caches are synced" controller="egressservices" I0331 03:58:54.199115 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0331 03:58:54.199120 3934 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0331 03:58:54.199125 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0331 03:58:54.199131 3934 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0331 03:58:54.199136 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0331 03:58:54.199140 3934 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0331 03:58:54.199143 3934 egressservice_zone.go:228] Repairing Egress Services I0331 03:58:54.199389 3934 master_controller.go:88] Starting Admin Policy Based Route Controller I0331 03:58:54.199400 3934 external_controller.go:278] Starting Admin Policy Based Route Controller I0331 03:58:54.199407 3934 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-29.ec2.internal I0331 03:58:54.199414 3934 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-29.ec2.internal: 9.395µs I0331 03:58:54.199421 3934 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-50.ec2.internal I0331 03:58:54.199429 3934 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-50.ec2.internal: 8.562µs I0331 03:58:54.199434 3934 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-235.ec2.internal I0331 03:58:54.199437 3934 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-235.ec2.internal: 3.714µs I0331 03:58:54.199681 3934 default_network_controller.go:568] Completing all the Watchers took 308.001633ms I0331 03:58:54.199697 3934 default_network_controller.go:572] Starting unidling controllers I0331 03:58:54.199709 3934 unidle.go:46] Registering OVN SB ControllerEvent handler I0331 03:58:54.199714 3934 unidle.go:63] Populating Initial ContollerEvent events I0331 03:58:54.199725 3934 unidle.go:79] Setting up event handlers for services I0331 03:58:54.199772 3934 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0331 03:58:54.205714 3934 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0331 03:58:54.499658 3934 default_node_network_controller.go:518] Node connection status = connected I0331 03:58:54.515455 3934 bridgeconfig.go:537] Gateway is ready I0331 03:58:54.515470 3934 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0331 03:58:54.525778 3934 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.29. Updating OVN encap IP. I0331 03:58:54.530583 3934 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.29 - it is already configured I0331 03:58:54.530595 3934 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.29/24:{} 172.20.0.1/32:{}]. Updating annotations. I0331 03:58:54.530779 3934 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.29/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-29.ec2.internal","mac-address":"12:c7:7a:4d:c1:cd","ip-addresses":["10.0.0.29/24"],"ip-address":"10.0.0.29/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:bfb9571c-b847-4d90-9834-6f5292f11107 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.29/24"}] on node ip-10-0-0-29.ec2.internal I0331 03:58:54.561371 3934 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-29.ec2.internal in network "default" I0331 03:58:54.561387 3934 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-29.ec2.internal I0331 03:58:54.561398 3934 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-29.ec2.internal: 15.461µs I0331 03:58:54.561412 3934 egressfirewall.go:628] Syncing node "ip-10-0-0-29.ec2.internal" for egress firewall I0331 03:58:54.561429 3934 node_tracker.go:169] Node ip-10-0-0-29.ec2.internal switch + router changed, syncing services in network "default" I0331 03:58:54.561431 3934 master.go:532] Adding or Updating local node "ip-10-0-0-29.ec2.internal" for network "default" I0331 03:58:54.561442 3934 services_controller.go:574] Full service sync requested for network=default W0331 03:58:54.561587 3934 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:54.561601 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:54.561607 3934 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:54.561820 3934 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:58:54.561958 3934 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:58:54.565284 3934 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-29.ec2.internal. Updating no re-route policies I0331 03:58:54.565354 3934 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-29.ec2.internal I0331 03:58:54.565369 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-fsp6h I0331 03:58:54.565376 3934 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-29.ec2.internal I0331 03:58:54.565381 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-fsp6h I0331 03:58:54.565402 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-fsp6h after 0 failed attempt(s) I0331 03:58:54.565408 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-2wv6z I0331 03:58:54.565417 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-mqs7g I0331 03:58:54.565424 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-29.ec2.internal I0331 03:58:54.565425 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-2wv6z I0331 03:58:54.565430 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-mqs7g I0331 03:58:54.565431 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-29.ec2.internal I0331 03:58:54.565435 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-bjw5c I0331 03:58:54.565451 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-bjw5c I0331 03:58:54.565452 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-29.ec2.internal after 0 failed attempt(s) I0331 03:58:54.565450 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-cv5zx I0331 03:58:54.565465 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-m952r I0331 03:58:54.565466 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-b77db I0331 03:58:54.565472 3934 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-dz4n2 I0331 03:58:54.565473 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-cv5zx I0331 03:58:54.565470 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-6b6dn I0331 03:58:54.565477 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-b77db I0331 03:58:54.565475 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-bjw5c after 0 failed attempt(s) I0331 03:58:54.565478 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-hjsvb I0331 03:58:54.565479 3934 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-dz4n2 I0331 03:58:54.565484 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-6b6dn I0331 03:58:54.565492 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-hjsvb I0331 03:58:54.565490 3934 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.0.29 I0331 03:58:54.565495 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-cv5zx after 0 failed attempt(s) I0331 03:58:54.565495 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-b77db after 0 failed attempt(s) I0331 03:58:54.565403 3934 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-29.ec2.internal after 0 failed attempt(s) I0331 03:58:54.565455 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-mqs7g after 0 failed attempt(s) I0331 03:58:54.565513 3934 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-6b6dn] creating logical port openshift-network-diagnostics_network-check-target-6b6dn for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:54.565514 3934 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-dz4n2] creating logical port kube-system_global-pull-secret-syncer-dz4n2 for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:54.565475 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-m952r I0331 03:58:54.565521 3934 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-hjsvb] creating logical port openshift-multus_network-metrics-daemon-hjsvb for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:54.565534 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-m952r after 0 failed attempt(s) I0331 03:58:54.565474 3934 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-zsb55 I0331 03:58:54.565545 3934 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-zsb55 I0331 03:58:54.565354 3934 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-k2dvs I0331 03:58:54.565560 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-zsb55 after 0 failed attempt(s) I0331 03:58:54.565570 3934 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-k2dvs I0331 03:58:54.565579 3934 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-29.ec2.internal I0331 03:58:54.565609 3934 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-k2dvs after 0 failed attempt(s) I0331 03:58:54.565451 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-2wv6z after 0 failed attempt(s) I0331 03:58:54.566271 3934 pods.go:271] [openshift-multus/network-metrics-daemon-hjsvb] addLogicalPort took 758.181µs, libovsdb time 307.393µs I0331 03:58:54.566283 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-hjsvb after 0 failed attempt(s) I0331 03:58:54.566344 3934 pods.go:271] [kube-system/global-pull-secret-syncer-dz4n2] addLogicalPort took 840.947µs, libovsdb time 367.036µs I0331 03:58:54.566352 3934 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-dz4n2 after 0 failed attempt(s) I0331 03:58:54.566523 3934 pods.go:271] [openshift-network-diagnostics/network-check-target-6b6dn] addLogicalPort took 1.013491ms, libovsdb time 493.378µs I0331 03:58:54.566538 3934 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-6b6dn after 0 failed attempt(s) I0331 03:58:54.600594 3934 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0331 03:58:54.611582 3934 iptables.go:108] Creating table: mangle chain: OUTPUT I0331 03:58:54.615169 3934 iptables.go:108] Creating table: nat chain: OUTPUT I0331 03:58:54.633217 3934 iptables.go:108] Creating table: nat chain: PREROUTING I0331 03:58:54.714715 3934 iptables.go:108] Creating table: filter chain: FORWARD I0331 03:58:54.732385 3934 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0331 03:58:54.732497 3934 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0331 03:58:54.732563 3934 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0331 03:58:54.732645 3934 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0331 03:58:54.732698 3934 gateway_init.go:325] Gateway and management port readiness took 737.344484ms I0331 03:58:54.732721 3934 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0331 03:58:54.732769 3934 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0331 03:58:54.732782 3934 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0331 03:58:54.732791 3934 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0331 03:58:54.732852 3934 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0331 03:58:54.732967 3934 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 51.818µs I0331 03:58:54.733224 3934 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0331 03:58:54.733253 3934 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 19.381µs I0331 03:58:54.733261 3934 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 38.562µs I0331 03:58:54.733267 3934 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0331 03:58:54.733276 3934 nad_controller.go:173] [node-nad-controller NAD controller]: started I0331 03:58:54.733293 3934 default_node_network_controller.go:898] Starting the default node network controller I0331 03:58:54.733298 3934 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0331 03:58:54.733371 3934 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 13.237µs I0331 03:58:54.733455 3934 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0331 03:58:54.733463 3934 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0331 03:58:54.733486 3934 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b8e1c0 0xc0002ecb60 0xc0002ecbd0 0xc0002ecc40 0xc0002ecd20 0xc0002ecd90 0xc0002ece70 0xc000a2f590 0xc0002ecf50 0xc0002ecfc0 0xc00055a410 0xc00657cfc0 0xc0002ed0a0 map[0x2f67040:0xc003e078c0 0x2f67580:0xc003e07bc0 0x2f68000:0xc003e07800 0x2f682a0:0xc003e07860 0x2f687e0:0xc003e07920 0x2f68a80:0xc003e07980 0x2f68d20:0xc003e079e0 0x2f69500:0xc003e07c20 0x2f697a0:0xc003e07c80 0x2f93d40:0xc000a65800 0x2f940e0:0xc000a65860 0x2f94bc0:0xc000a7d2c0 0x2f96520:0xc001b7e000 0x2f97000:0xc00281f680 0x2f97e80:0xc00109b7a0] 0xc0000d2bd0 0} I0331 03:58:54.733549 3934 gateway.go:151] Starting gateway service sync I0331 03:58:54.734092 3934 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0331 03:58:54.736413 3934 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0331 03:58:54.764049 3934 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0331 03:58:54.765353 3934 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0331 03:58:54.766593 3934 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0331 03:58:54.801326 3934 gateway.go:168] Gateway service sync done. Time taken: 67.764984ms I0331 03:58:54.802102 3934 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30871 I0331 03:58:54.802478 3934 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b8e1c0 0xc0002ecb60 0xc0002ecbd0 0xc0002ecc40 0xc0002ecd20 0xc0002ecd90 0xc0002ece70 0xc000a2f590 0xc0002ecf50 0xc0002ecfc0 0xc00055a410 0xc00657cfc0 0xc0002ed0a0 map[0x2f67040:0xc003e078c0 0x2f67580:0xc003e07bc0 0x2f68000:0xc003e07800 0x2f682a0:0xc003e07860 0x2f687e0:0xc003e07920 0x2f68a80:0xc003e07980 0x2f68d20:0xc003e079e0 0x2f69500:0xc003e07c20 0x2f697a0:0xc003e07c80 0x2f93d40:0xc000a65800 0x2f940e0:0xc000a65860 0x2f94bc0:0xc000a7d2c0 0x2f96520:0xc001b7e000 0x2f97000:0xc00281f680 0x2f97e80:0xc00109b7a0] 0xc0000d2bd0 0} I0331 03:58:54.803291 3934 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b8e1c0 0xc0002ecb60 0xc0002ecbd0 0xc0002ecc40 0xc0002ecd20 0xc0002ecd90 0xc0002ece70 0xc000a2f590 0xc0002ecf50 0xc0002ecfc0 0xc00055a410 0xc00657cfc0 0xc0002ed0a0 map[0x2f67040:0xc003e078c0 0x2f67580:0xc003e07bc0 0x2f68000:0xc003e07800 0x2f682a0:0xc003e07860 0x2f687e0:0xc003e07920 0x2f68a80:0xc003e07980 0x2f68d20:0xc003e079e0 0x2f69500:0xc003e07c20 0x2f697a0:0xc003e07c80 0x2f93d40:0xc000a65800 0x2f940e0:0xc000a65860 0x2f94bc0:0xc000a7d2c0 0x2f96520:0xc001b7e000 0x2f97000:0xc00281f680 0x2f97e80:0xc00109b7a0] 0xc0000d2bd0 0} I0331 03:58:54.803355 3934 gateway.go:519] Reconciling gateway with updates I0331 03:58:54.809015 3934 gateway.go:326] Spawning Conntrack Rule Check Thread I0331 03:58:54.809033 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:58:54.809040 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/console I0331 03:58:54.809046 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:58:54.809052 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/console I0331 03:58:54.809062 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:58:54.809072 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0331 03:58:54.809075 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:58:54.809085 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0331 03:58:54.809088 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:58:54.809090 3934 default_node_network_controller.go:943] Gateway and management port readiness took 75.62098ms I0331 03:58:54.809091 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:58:54.809106 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:58:54.809109 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:58:54.809110 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/console after 0 failed attempt(s) I0331 03:58:54.809088 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0331 03:58:54.809077 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:58:54.809123 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/exporter I0331 03:58:54.809130 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0331 03:58:54.809137 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:58:54.809136 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:58:54.809140 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/exporter I0331 03:58:54.809144 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0331 03:58:54.809143 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0331 03:58:54.809065 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:58:54.809153 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:58:54.809147 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:58:54.809161 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0331 03:58:54.809164 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:58:54.809161 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:58:54.809170 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:58:54.809173 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0331 03:58:54.809176 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:58:54.809179 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:58:54.809169 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:58:54.809182 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0331 03:58:54.809179 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:58:54.809188 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:58:54.809184 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:58:54.809193 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:58:54.809195 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:58:54.809195 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:58:54.809195 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:58:54.809207 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:58:54.809210 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:58:54.809228 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0331 03:58:54.809234 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:58:54.809235 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:58:54.809241 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:58:54.809245 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:58:54.809244 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:58:54.809133 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0331 03:58:54.809252 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0331 03:58:54.809250 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:58:54.809258 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0331 03:58:54.809261 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0331 03:58:54.809260 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:58:54.809266 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:58:54.809266 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0331 03:58:54.809271 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0331 03:58:54.809274 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:58:54.809249 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:58:54.809289 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0331 03:58:54.809206 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:58:54.809304 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0331 03:58:54.809310 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:58:54.809145 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:58:54.809311 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0331 03:58:54.809323 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:58:54.809237 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/downloads I0331 03:58:54.809334 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/downloads I0331 03:58:54.809202 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:58:54.809265 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:58:54.809400 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0331 03:58:54.809092 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0331 03:58:54.809213 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0331 03:58:54.809416 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0331 03:58:54.809427 3934 port_claim.go:71] Svc openshift-ingress/router-default: port 30718 is already open, no action needed I0331 03:58:54.809435 3934 port_claim.go:71] Svc openshift-ingress/router-default: port 31101 is already open, no action needed I0331 03:58:54.809106 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:58:54.809488 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:58:54.809494 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0331 03:58:54.809120 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:58:54.809502 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:58:54.809505 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0331 03:58:54.809120 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:58:54.809513 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:58:54.809516 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0331 03:58:54.809134 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:58:54.809525 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:58:54.809139 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:58:54.809146 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:58:54.809574 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0331 03:58:54.809150 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0331 03:58:54.809152 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/exporter after 0 failed attempt(s) I0331 03:58:54.809149 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:58:54.809587 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:58:54.809089 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:58:54.809613 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:58:54.809152 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:58:54.809626 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:58:54.809158 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0331 03:58:54.809097 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0331 03:58:54.809163 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:58:54.809162 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:58:54.809727 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:58:54.809164 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:58:54.809753 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:58:54.809759 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0331 03:58:54.809182 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:58:54.809767 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:58:54.809185 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:58:54.809791 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:58:54.809184 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0331 03:58:54.809220 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:58:54.809223 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:58:54.809841 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:58:54.809220 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0331 03:58:54.809236 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook after 0 failed attempt(s) I0331 03:58:54.809238 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0331 03:58:54.809250 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0331 03:58:54.809258 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0331 03:58:54.809255 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0331 03:58:54.809935 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0331 03:58:54.809296 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0331 03:58:54.809361 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:58:54.810157 3934 node_ip_handler_linux.go:160] Node IP manager is running W0331 03:58:54.845301 3934 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0331 03:58:54.845475 3934 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0331 03:58:54.845598 3934 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0331 03:58:54.854204 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0331 03:58:54.854231 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0331 03:58:54.854242 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0331 03:58:54.854248 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0331 03:58:54.854255 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/downloads after 0 failed attempt(s) I0331 03:58:54.854261 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0331 03:58:54.854267 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0331 03:58:54.891847 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0331 03:58:54.891855 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0331 03:58:54.891871 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0331 03:58:54.891881 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0331 03:58:54.891905 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0331 03:58:54.891915 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0331 03:58:54.891922 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0331 03:58:54.891927 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0331 03:58:54.891933 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0331 03:58:54.891957 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0331 03:58:54.891967 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0331 03:58:54.891982 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0331 03:58:54.929105 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0331 03:58:54.929126 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0331 03:58:54.929149 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0331 03:58:56.073483 3934 default_node_network_controller.go:1364] Starting node controller node sync W0331 03:58:56.073517 3934 egressip_healthcheck.go:76] Health checking using insecure connection I0331 03:58:56.073592 3934 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0331 03:58:56.085391 3934 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.889519ms I0331 03:58:56.085530 3934 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-50.ec2.internal", IP: 10.0.1.50 to PMTUD blocking rules I0331 03:58:56.085545 3934 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-50.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:58:56.091450 3934 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0331 03:58:56.091719 3934 egressservice_node.go:93] Setting up event handlers for Egress Services I0331 03:58:56.091933 3934 egressservice_node.go:183] Starting Egress Services Controller I0331 03:58:56.091950 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0331 03:58:56.091960 3934 shared_informer.go:356] "Caches are synced" controller="egressservices" I0331 03:58:56.091965 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0331 03:58:56.091970 3934 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0331 03:58:56.091975 3934 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0331 03:58:56.091980 3934 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0331 03:58:56.091984 3934 egressservice_node.go:197] Repairing Egress Services I0331 03:58:56.101076 3934 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0331 03:58:56.101086 3934 external_controller.go:278] Starting Admin Policy Based Route Controller I0331 03:58:56.101395 3934 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0331 03:58:56.101406 3934 link_network_manager.go:117] Link manager is running I0331 03:58:56.101414 3934 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0331 03:58:56.101479 3934 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0331 03:58:56.102086 3934 vrf_manager.go:128] VRF manager is running I0331 03:58:56.102161 3934 node_controller_manager.go:437] Removing flows to drop GARP I0331 03:58:56.102170 3934 gateway.go:519] Reconciling gateway with updates I0331 03:58:56.117971 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:58:56.117992 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0331 03:58:56.118007 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0331 03:58:56.118019 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0331 03:58:56.118032 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:58:56.118045 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0331 03:58:56.118062 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0331 03:58:56.118062 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:58:56.118075 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:58:56.118081 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0331 03:58:56.118082 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:58:56.118088 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0331 03:58:56.118085 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0331 03:58:56.118089 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:58:56.118096 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:58:56.118097 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0331 03:58:56.118103 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:58:56.118106 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0331 03:58:56.118108 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0331 03:58:56.118111 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:58:56.118111 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:58:56.118121 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0331 03:58:56.118126 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0331 03:58:56.118128 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0331 03:58:56.118136 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0331 03:58:56.118133 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0331 03:58:56.118150 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0331 03:58:56.118154 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0331 03:58:56.118157 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/exporter I0331 03:58:56.118084 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0331 03:58:56.118161 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0331 03:58:56.118159 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:58:56.118168 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0331 03:58:56.118176 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:58:56.118160 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0331 03:58:56.118179 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0331 03:58:56.118184 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0331 03:58:56.118184 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0331 03:58:56.118176 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:58:56.118186 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:58:56.118201 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0331 03:58:56.118200 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0331 03:58:56.118204 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0331 03:58:56.118215 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0331 03:58:56.118221 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0331 03:58:56.118229 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0331 03:58:56.118229 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0331 03:58:56.118230 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0331 03:58:56.118168 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/exporter I0331 03:58:56.118247 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:58:56.118247 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0331 03:58:56.118252 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/exporter after 0 failed attempt(s) I0331 03:58:56.118234 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0331 03:58:56.118261 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0331 03:58:56.118172 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0331 03:58:56.118261 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0331 03:58:56.118305 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:58:56.118318 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0331 03:58:56.118350 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:58:56.118357 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:58:56.118365 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0331 03:58:56.118369 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:58:56.118372 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0331 03:58:56.118376 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:58:56.118378 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:58:56.118382 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0331 03:58:56.118381 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0331 03:58:56.118383 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:58:56.118389 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0331 03:58:56.118386 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0331 03:58:56.118394 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:58:56.118394 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0331 03:58:56.118394 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0331 03:58:56.118397 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0331 03:58:56.118404 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0331 03:58:56.118414 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:58:56.118423 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0331 03:58:56.118431 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0331 03:58:56.118442 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0331 03:58:56.118441 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:58:56.118445 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/downloads I0331 03:58:56.118449 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook I0331 03:58:56.118444 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:58:56.118454 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/downloads I0331 03:58:56.118455 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0331 03:58:56.118458 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0331 03:58:56.118461 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:58:56.118465 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0331 03:58:56.118462 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:58:56.118468 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:58:56.118475 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0331 03:58:56.118474 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:58:56.118469 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0331 03:58:56.118482 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0331 03:58:56.118483 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:58:56.118486 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console/console I0331 03:58:56.118477 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:58:56.118495 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0331 03:58:56.118497 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console/console I0331 03:58:56.118501 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0331 03:58:56.118498 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:58:56.118500 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:58:56.118500 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0331 03:58:56.118516 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 03:58:56.118516 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0331 03:58:56.118517 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0331 03:58:56.118488 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0331 03:58:56.118453 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0331 03:58:56.118479 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0331 03:58:56.118568 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:58:56.118582 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0331 03:58:56.118624 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:58:56.118628 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0331 03:58:56.118632 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0331 03:58:56.118627 3934 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:58:56.118636 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0331 03:58:56.118641 3934 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0331 03:58:56.118645 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0331 03:58:56.118650 3934 port_claim.go:71] Svc openshift-ingress/router-default: port 30718 is already open, no action needed I0331 03:58:56.118665 3934 port_claim.go:71] Svc openshift-ingress/router-default: port 31101 is already open, no action needed I0331 03:58:56.140743 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0331 03:58:56.140762 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0331 03:58:56.140786 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0331 03:58:56.140793 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0331 03:58:56.140798 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0331 03:58:56.140803 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0331 03:58:56.140813 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0331 03:58:56.140819 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0331 03:58:56.140824 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/prometheus-operator-admission-webhook after 0 failed attempt(s) I0331 03:58:56.140829 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0331 03:58:56.140834 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0331 03:58:56.140839 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/downloads after 0 failed attempt(s) I0331 03:58:56.140844 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0331 03:58:56.140849 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0331 03:58:56.140861 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0331 03:58:56.140868 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0331 03:58:56.140872 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0331 03:58:56.140877 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0331 03:58:56.177982 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0331 03:58:56.178003 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0331 03:58:56.178009 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console/console after 0 failed attempt(s) I0331 03:58:56.178015 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0331 03:58:56.178022 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0331 03:58:56.178029 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0331 03:58:56.214770 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0331 03:58:56.214777 3934 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0331 03:58:57.489807 3934 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-235.ec2.internal in network "default" I0331 03:58:57.489834 3934 node_tracker.go:235] Node ip-10-0-2-235.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-235.ec2.internal" I0331 03:58:57.489850 3934 node_tracker.go:169] Node ip-10-0-2-235.ec2.internal switch + router changed, syncing services in network "default" I0331 03:58:57.489864 3934 services_controller.go:574] Full service sync requested for network=default I0331 03:58:57.489852 3934 default_network_controller.go:1008] Node "ip-10-0-2-235.ec2.internal" in remote zone "ip-10-0-2-235.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0331 03:58:57.491300 3934 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:58:57.491751 3934 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:58:57.491979 3934 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:57.492025 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:57.492055 3934 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0331 03:58:57.586121 3934 default_network_controller.go:1008] Node "ip-10-0-2-235.ec2.internal" in remote zone "ip-10-0-2-235.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:58:57.607025 3934 default_network_controller.go:1008] Node "ip-10-0-2-235.ec2.internal" in remote zone "ip-10-0-2-235.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0331 03:58:57.607035 3934 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-235.ec2.internal in network "default" I0331 03:58:57.607092 3934 node_tracker.go:169] Node ip-10-0-2-235.ec2.internal switch + router changed, syncing services in network "default" I0331 03:58:57.607107 3934 services_controller.go:574] Full service sync requested for network=default W0331 03:58:57.607296 3934 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:58:57.607370 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:57.607384 3934 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:57.607388 3934 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:57.607562 3934 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0331 03:58:57.607750 3934 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-235.ec2.internal for the network default I0331 03:58:57.608849 3934 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-235.ec2.internal" on network "default" took: 1.111739ms I0331 03:58:57.980876 3934 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-29.ec2.internal I0331 03:58:57.980922 3934 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-29.ec2.internal: 49.774µs W0331 03:58:58.004216 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0331 03:58:58.022942 3934 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-58586876-h272k] creating logical port openshift-image-registry_image-registry-58586876-h272k for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:58.023063 3934 kube.go:256] Updating pod openshift-image-registry/image-registry-58586876-h272k I0331 03:58:58.024382 3934 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-xvhkz] creating logical port openshift-console_downloads-56d45c498b-xvhkz for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:58.024464 3934 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-xvhkz I0331 03:58:58.042094 3934 pod.go:62] [openshift-console/downloads-56d45c498b-xvhkz] pod update took 17.647217ms I0331 03:58:58.042111 3934 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-xvhkz] addLogicalPort annotation time took 17.66772ms I0331 03:58:58.043067 3934 pods.go:271] [openshift-console/downloads-56d45c498b-xvhkz] addLogicalPort took 18.692229ms, libovsdb time 746.945µs I0331 03:58:58.044916 3934 pod.go:62] [openshift-image-registry/image-registry-58586876-h272k] pod update took 21.879911ms I0331 03:58:58.044936 3934 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-58586876-h272k] addLogicalPort annotation time took 21.900452ms I0331 03:58:58.045799 3934 pods.go:271] [openshift-image-registry/image-registry-58586876-h272k] addLogicalPort took 22.867394ms, libovsdb time 663.778µs W0331 03:58:58.049487 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0331 03:58:58.049582 3934 base_network_controller_pods.go:487] [default/openshift-console/console-ff4b454d6-cq7hv] creating logical port openshift-console_console-ff4b454d6-cq7hv for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:58.049675 3934 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zcmvl] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-zcmvl for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:58.049808 3934 kube.go:256] Updating pod openshift-console/console-ff4b454d6-cq7hv I0331 03:58:58.049818 3934 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zcmvl I0331 03:58:58.068086 3934 pod.go:62] [openshift-console/console-ff4b454d6-cq7hv] pod update took 18.306759ms I0331 03:58:58.068108 3934 base_network_controller_pods.go:937] [default/openshift-console/console-ff4b454d6-cq7hv] addLogicalPort annotation time took 18.33043ms I0331 03:58:58.068128 3934 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zcmvl] pod update took 18.329178ms I0331 03:58:58.068140 3934 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zcmvl] addLogicalPort annotation time took 18.341915ms I0331 03:58:58.069092 3934 pods.go:271] [openshift-console/console-ff4b454d6-cq7hv] addLogicalPort took 19.523404ms, libovsdb time 680.149µs I0331 03:58:58.069375 3934 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zcmvl] addLogicalPort took 19.7131ms, libovsdb time 951.449µs I0331 03:58:58.073299 3934 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-49qnd] creating logical port openshift-dns_dns-default-49qnd for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:58.073379 3934 kube.go:256] Updating pod openshift-dns/dns-default-49qnd I0331 03:58:58.076803 3934 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-76cbd74c4-tsvll] creating logical port openshift-ingress_router-default-76cbd74c4-tsvll for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:58.076913 3934 kube.go:256] Updating pod openshift-ingress/router-default-76cbd74c4-tsvll I0331 03:58:58.082778 3934 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-m4ng4] creating logical port openshift-insights_insights-runtime-extractor-m4ng4 for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:58.082839 3934 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-m4ng4 I0331 03:58:58.094686 3934 pod.go:62] [openshift-dns/dns-default-49qnd] pod update took 21.33559ms I0331 03:58:58.094707 3934 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-49qnd] addLogicalPort annotation time took 21.357331ms I0331 03:58:58.095629 3934 pods.go:271] [openshift-dns/dns-default-49qnd] addLogicalPort took 22.339625ms, libovsdb time 671.323µs I0331 03:58:58.101550 3934 pod.go:62] [openshift-insights/insights-runtime-extractor-m4ng4] pod update took 18.730761ms I0331 03:58:58.101566 3934 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-m4ng4] addLogicalPort annotation time took 18.748873ms I0331 03:58:58.101690 3934 pod.go:62] [openshift-ingress/router-default-76cbd74c4-tsvll] pod update took 24.818442ms I0331 03:58:58.101708 3934 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-76cbd74c4-tsvll] addLogicalPort annotation time took 24.840346ms I0331 03:58:58.102520 3934 pods.go:271] [openshift-insights/insights-runtime-extractor-m4ng4] addLogicalPort took 19.767067ms, libovsdb time 692.366µs I0331 03:58:58.103046 3934 pods.go:271] [openshift-ingress/router-default-76cbd74c4-tsvll] addLogicalPort took 26.256196ms, libovsdb time 1.073433ms W0331 03:58:58.120944 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:58.120962 3934 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:58.120969 3934 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default I0331 03:58:58.149955 3934 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-9tb58] creating logical port openshift-ingress-canary_ingress-canary-9tb58 for pod on switch ip-10-0-0-29.ec2.internal I0331 03:58:58.150035 3934 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-9tb58 I0331 03:58:58.164967 3934 pod.go:62] [openshift-ingress-canary/ingress-canary-9tb58] pod update took 14.952127ms I0331 03:58:58.164984 3934 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-9tb58] addLogicalPort annotation time took 14.973383ms I0331 03:58:58.165964 3934 pods.go:271] [openshift-ingress-canary/ingress-canary-9tb58] addLogicalPort took 16.023547ms, libovsdb time 732.73µs I0331 03:58:58.440058 3934 cni.go:403] [openshift-image-registry/image-registry-58586876-h272k 305ace43209eca308ef93b906c8bd39d7dae806c8b901650c462adc7a79884e9 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-58586876-h272k 305ace43209eca308ef93b906c8bd39d7dae806c8b901650c462adc7a79884e9 network default NAD default] I0331 03:58:58.458796 3934 cni.go:403] [openshift-console/downloads-56d45c498b-xvhkz 37edb388258279b1baf77bac33ded55143719c33e4a25bc7e00c884ee3a1172a network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-xvhkz 37edb388258279b1baf77bac33ded55143719c33e4a25bc7e00c884ee3a1172a network default NAD default] I0331 03:58:58.478573 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-58586876-h272k, hostIfaceName: 305ace43209eca3, network: default, NAD default, SandboxID: "305ace43209eca308ef93b906c8bd39d7dae806c8b901650c462adc7a79884e9", PCI device ID: , UID: "63624670-a2f5-4f12-acd8-cf0e48be9209", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0331 03:58:58.478840 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-xvhkz, hostIfaceName: 37edb388258279b, network: default, NAD default, SandboxID: "37edb388258279b1baf77bac33ded55143719c33e4a25bc7e00c884ee3a1172a", PCI device ID: , UID: "9d38e134-65d9-4f89-aa37-b07d7d32acd5", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0331 03:58:58.494545 3934 cni.go:403] [openshift-console/console-ff4b454d6-cq7hv ebd643c5194167cd9b391df11976ad76ab10f1d35774a5d53e6a6e40f70a11c7 network default NAD default] ADD starting CNI request [openshift-console/console-ff4b454d6-cq7hv ebd643c5194167cd9b391df11976ad76ab10f1d35774a5d53e6a6e40f70a11c7 network default NAD default] I0331 03:58:58.504496 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-ff4b454d6-cq7hv, hostIfaceName: ebd643c5194167c, network: default, NAD default, SandboxID: "ebd643c5194167cd9b391df11976ad76ab10f1d35774a5d53e6a6e40f70a11c7", PCI device ID: , UID: "7c6a32eb-5d5b-4fac-8e11-11e27045ca08", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0331 03:58:58.531164 3934 cni.go:403] [openshift-dns/dns-default-49qnd 422c9fc4ca267dd2062c502ee319f84a3f4fe5511ab6a228d16c088e84ee44e6 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-49qnd 422c9fc4ca267dd2062c502ee319f84a3f4fe5511ab6a228d16c088e84ee44e6 network default NAD default] I0331 03:58:58.535208 3934 cni.go:424] [openshift-image-registry/image-registry-58586876-h272k 305ace43209eca308ef93b906c8bd39d7dae806c8b901650c462adc7a79884e9 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-58586876-h272k 305ace43209eca308ef93b906c8bd39d7dae806c8b901650c462adc7a79884e9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:3f:54:25:64:a5\",\"name\":\"305ace43209eca3\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/157fcd62-602b-435a-8b36-8b1a1368afb9\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:58:58.536864 3934 cni.go:424] [openshift-console/downloads-56d45c498b-xvhkz 37edb388258279b1baf77bac33ded55143719c33e4a25bc7e00c884ee3a1172a network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-xvhkz 37edb388258279b1baf77bac33ded55143719c33e4a25bc7e00c884ee3a1172a network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:71:9a:13:b2:04\",\"name\":\"37edb388258279b\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55b5b8fc-dc05-4dad-b779-29ad8a19ba58\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:58:58.543334 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-49qnd, hostIfaceName: 422c9fc4ca267dd, network: default, NAD default, SandboxID: "422c9fc4ca267dd2062c502ee319f84a3f4fe5511ab6a228d16c088e84ee44e6", PCI device ID: , UID: "aeb120e8-7aa8-47da-9c57-63719d8627cb", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0331 03:58:58.552108 3934 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zcmvl cdf9afe61c549ca953fb99003921fa4262d1226908e4e7e8f77e2161542025b8 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zcmvl cdf9afe61c549ca953fb99003921fa4262d1226908e4e7e8f77e2161542025b8 network default NAD default] I0331 03:58:58.558048 3934 cni.go:424] [openshift-console/console-ff4b454d6-cq7hv ebd643c5194167cd9b391df11976ad76ab10f1d35774a5d53e6a6e40f70a11c7 network default NAD default] ADD finished CNI request [openshift-console/console-ff4b454d6-cq7hv ebd643c5194167cd9b391df11976ad76ab10f1d35774a5d53e6a6e40f70a11c7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:e9:6d:da:21:d6\",\"name\":\"ebd643c5194167c\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fea9f078-f366-4c81-84a1-389fc8bf56d2\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:58:58.562221 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-zcmvl, hostIfaceName: cdf9afe61c549ca, network: default, NAD default, SandboxID: "cdf9afe61c549ca953fb99003921fa4262d1226908e4e7e8f77e2161542025b8", PCI device ID: , UID: "9218e4c4-1041-4cee-90e5-3ce245cdd433", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0331 03:58:58.570442 3934 cni.go:403] [openshift-ingress/router-default-76cbd74c4-tsvll 66cb636948807330a56489965b17be36abf726cfae09c47c7ba4125d66dc332c network default NAD default] ADD starting CNI request [openshift-ingress/router-default-76cbd74c4-tsvll 66cb636948807330a56489965b17be36abf726cfae09c47c7ba4125d66dc332c network default NAD default] I0331 03:58:58.582283 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-76cbd74c4-tsvll, hostIfaceName: 66cb63694880733, network: default, NAD default, SandboxID: "66cb636948807330a56489965b17be36abf726cfae09c47c7ba4125d66dc332c", PCI device ID: , UID: "d647e61c-23ab-4313-a3aa-57c4a168c9c7", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0331 03:58:58.592312 3934 cni.go:403] [openshift-ingress-canary/ingress-canary-9tb58 a767e0473285721b36f1997ac2a2aa7df9dd7f978cf4f601511b38b076ff07db network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-9tb58 a767e0473285721b36f1997ac2a2aa7df9dd7f978cf4f601511b38b076ff07db network default NAD default] I0331 03:58:58.599969 3934 cni.go:424] [openshift-dns/dns-default-49qnd 422c9fc4ca267dd2062c502ee319f84a3f4fe5511ab6a228d16c088e84ee44e6 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-49qnd 422c9fc4ca267dd2062c502ee319f84a3f4fe5511ab6a228d16c088e84ee44e6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:a8:47:dd:d6:71\",\"name\":\"422c9fc4ca267dd\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9f0cdbd6-ab18-42a4-94b2-68cf1f14501b\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:58:58.605540 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-9tb58, hostIfaceName: a767e0473285721, network: default, NAD default, SandboxID: "a767e0473285721b36f1997ac2a2aa7df9dd7f978cf4f601511b38b076ff07db", PCI device ID: , UID: "d9541337-71ea-4732-8ac8-4426f9d12ac4", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0331 03:58:58.618251 3934 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zcmvl cdf9afe61c549ca953fb99003921fa4262d1226908e4e7e8f77e2161542025b8 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-zcmvl cdf9afe61c549ca953fb99003921fa4262d1226908e4e7e8f77e2161542025b8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:97:ea:e5:98:84\",\"name\":\"cdf9afe61c549ca\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/69c9d47b-1306-480c-9c03-554efaf2a998\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:58:58.621511 3934 cni.go:403] [openshift-insights/insights-runtime-extractor-m4ng4 a3466301ae2212bbdf7146ed46d4249fb6276423c6ae506c5e5834fe4087bd2b network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-m4ng4 a3466301ae2212bbdf7146ed46d4249fb6276423c6ae506c5e5834fe4087bd2b network default NAD default] I0331 03:58:58.632769 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-m4ng4, hostIfaceName: a3466301ae2212b, network: default, NAD default, SandboxID: "a3466301ae2212bbdf7146ed46d4249fb6276423c6ae506c5e5834fe4087bd2b", PCI device ID: , UID: "34920b09-a921-4b48-a176-23e4521bf183", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0331 03:58:58.634599 3934 cni.go:424] [openshift-ingress/router-default-76cbd74c4-tsvll 66cb636948807330a56489965b17be36abf726cfae09c47c7ba4125d66dc332c network default NAD default] ADD finished CNI request [openshift-ingress/router-default-76cbd74c4-tsvll 66cb636948807330a56489965b17be36abf726cfae09c47c7ba4125d66dc332c network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:47:ba:8e:a9:d2\",\"name\":\"66cb63694880733\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a0b8206a-5f93-4d0a-908f-77758ade10b6\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:58:58.654385 3934 cni.go:424] [openshift-ingress-canary/ingress-canary-9tb58 a767e0473285721b36f1997ac2a2aa7df9dd7f978cf4f601511b38b076ff07db network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-9tb58 a767e0473285721b36f1997ac2a2aa7df9dd7f978cf4f601511b38b076ff07db network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:38:d3:fd:4d:10\",\"name\":\"a767e0473285721\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7ff8f381-cdd3-4437-9a0e-05832b7b799f\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:58:58.675611 3934 cni.go:424] [openshift-insights/insights-runtime-extractor-m4ng4 a3466301ae2212bbdf7146ed46d4249fb6276423c6ae506c5e5834fe4087bd2b network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-m4ng4 a3466301ae2212bbdf7146ed46d4249fb6276423c6ae506c5e5834fe4087bd2b network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:40:f4:8e:f1:a8\",\"name\":\"a3466301ae2212b\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e87b7538-74b7-47ec-877f-fd1cbe58ccf4\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:58:58.681171 3934 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-235.ec2.internal I0331 03:58:58.681235 3934 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-235.ec2.internal: 69.696µs I0331 03:58:58.681175 3934 egressfirewall.go:628] Syncing node "ip-10-0-2-235.ec2.internal" for egress firewall I0331 03:58:58.681187 3934 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-235.ec2.internal in network "default" I0331 03:58:58.681256 3934 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-235.ec2.internal", IP: 10.0.2.235 to PMTUD blocking rules I0331 03:58:58.681268 3934 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-235.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0331 03:58:58.681312 3934 node_tracker.go:169] Node ip-10-0-2-235.ec2.internal switch + router changed, syncing services in network "default" I0331 03:58:58.681332 3934 services_controller.go:574] Full service sync requested for network=default W0331 03:58:58.681464 3934 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:58.681476 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:58.681487 3934 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:58:58.681568 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0331 03:58:58.681825 3934 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:58:58.687208 3934 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-235.ec2.internal. Updating no re-route policies I0331 03:58:58.699283 3934 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-235.ec2.internal", IP: 10.0.2.235 to PMTUD blocking rules I0331 03:58:58.699300 3934 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-235.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0331 03:58:58.820389 3934 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:58:59.834905 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0331 03:58:59.834944 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0331 03:59:00.840163 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:59:00.840192 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:59:00.840200 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0331 03:59:00.841290 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0331 03:59:00.841313 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0331 03:59:00.969781 3934 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0331 03:59:02.921101 3934 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-235.ec2.internal I0331 03:59:02.921136 3934 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-235.ec2.internal: 42.848µs W0331 03:59:03.864483 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0331 03:59:03.912739 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0331 03:59:03.941637 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:03.941657 3934 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:03.941662 3934 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:03.980164 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:03.980186 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:03.980194 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:03.980212 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:03.980220 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:03.980227 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:04.192925 3934 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0331 03:59:04.691512 3934 cni.go:403] [kube-system/global-pull-secret-syncer-dz4n2 8e13b245308e6b0e513369e44260d03794f397c5cc49e18330b0ccba89ffd8e8 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-dz4n2 8e13b245308e6b0e513369e44260d03794f397c5cc49e18330b0ccba89ffd8e8 network default NAD default] I0331 03:59:04.702757 3934 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-dz4n2, hostIfaceName: 8e13b245308e6b0, network: default, NAD default, SandboxID: "8e13b245308e6b0e513369e44260d03794f397c5cc49e18330b0ccba89ffd8e8", PCI device ID: , UID: "defd1772-5557-4591-a361-a3594db148d6", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0331 03:59:04.749520 3934 cni.go:424] [kube-system/global-pull-secret-syncer-dz4n2 8e13b245308e6b0e513369e44260d03794f397c5cc49e18330b0ccba89ffd8e8 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-dz4n2 8e13b245308e6b0e513369e44260d03794f397c5cc49e18330b0ccba89ffd8e8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:be:9d:c8:5f:9a\",\"name\":\"8e13b245308e6b0\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/362ef847-d507-4f9b-8401-bd5678ee6df6\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:59:04.846062 3934 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-c56ff685-xm52f] creating logical port openshift-monitoring_prometheus-operator-c56ff685-xm52f for pod on switch ip-10-0-0-29.ec2.internal I0331 03:59:04.846158 3934 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-c56ff685-xm52f I0331 03:59:04.865672 3934 pod.go:62] [openshift-monitoring/prometheus-operator-c56ff685-xm52f] pod update took 19.540963ms I0331 03:59:04.865694 3934 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-c56ff685-xm52f] addLogicalPort annotation time took 19.565218ms I0331 03:59:04.866940 3934 pods.go:271] [openshift-monitoring/prometheus-operator-c56ff685-xm52f] addLogicalPort took 20.894742ms, libovsdb time 931.773µs I0331 03:59:05.236666 3934 cni.go:403] [openshift-monitoring/prometheus-operator-c56ff685-xm52f a1c011f177a90e9488287cb6de834d7e185971070613da16090b8eb1cbfb73b1 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-c56ff685-xm52f a1c011f177a90e9488287cb6de834d7e185971070613da16090b8eb1cbfb73b1 network default NAD default] I0331 03:59:05.245930 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-c56ff685-xm52f, hostIfaceName: a1c011f177a90e9, network: default, NAD default, SandboxID: "a1c011f177a90e9488287cb6de834d7e185971070613da16090b8eb1cbfb73b1", PCI device ID: , UID: "b65c789f-8d90-4c33-8472-b054d88f30f5", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0331 03:59:05.293041 3934 cni.go:424] [openshift-monitoring/prometheus-operator-c56ff685-xm52f a1c011f177a90e9488287cb6de834d7e185971070613da16090b8eb1cbfb73b1 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-c56ff685-xm52f a1c011f177a90e9488287cb6de834d7e185971070613da16090b8eb1cbfb73b1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:02:e2:08:80:3f\",\"name\":\"a1c011f177a90e9\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fb6729cb-270e-440b-99da-44cd0adf2b3d\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 03:59:07.242186 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0331 03:59:10.099432 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0331 03:59:10.517117 3934 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-5qr6d] creating logical port openshift-monitoring_kube-state-metrics-664d8f7fc8-5qr6d for pod on switch ip-10-0-0-29.ec2.internal I0331 03:59:10.517240 3934 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-664d8f7fc8-5qr6d I0331 03:59:10.538119 3934 pod.go:62] [openshift-monitoring/kube-state-metrics-664d8f7fc8-5qr6d] pod update took 20.922232ms I0331 03:59:10.538142 3934 base_network_controller_pods.go:937] [default/openshift-monitoring/kube-state-metrics-664d8f7fc8-5qr6d] addLogicalPort annotation time took 20.95082ms I0331 03:59:10.539537 3934 pods.go:271] [openshift-monitoring/kube-state-metrics-664d8f7fc8-5qr6d] addLogicalPort took 22.436759ms, libovsdb time 1.000619ms I0331 03:59:11.492444 3934 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-0-29.ec2.internal I0331 03:59:11.492613 3934 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0331 03:59:11.519546 3934 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 26.973679ms I0331 03:59:11.519568 3934 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 26.999544ms I0331 03:59:11.520925 3934 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 28.49992ms, libovsdb time 1.010013ms I0331 03:59:12.451283 3934 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-57f477cc75-nwcqv] creating logical port openshift-monitoring_thanos-querier-57f477cc75-nwcqv for pod on switch ip-10-0-0-29.ec2.internal I0331 03:59:12.451391 3934 kube.go:256] Updating pod openshift-monitoring/thanos-querier-57f477cc75-nwcqv I0331 03:59:12.470219 3934 pod.go:62] [openshift-monitoring/thanos-querier-57f477cc75-nwcqv] pod update took 18.858695ms I0331 03:59:12.470242 3934 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-57f477cc75-nwcqv] addLogicalPort annotation time took 18.891904ms I0331 03:59:12.471456 3934 pods.go:271] [openshift-monitoring/thanos-querier-57f477cc75-nwcqv] addLogicalPort took 20.191279ms, libovsdb time 895.965µs W0331 03:59:12.957696 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:59:12.957726 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:59:13.716683 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:59:13.716706 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:59:13.857617 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:13.857635 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:13.857643 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0331 03:59:14.310627 3934 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:14.310645 3934 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:14.310651 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:14.337650 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:59:14.337702 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:59:14.379170 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0331 03:59:14.379195 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0331 03:59:15.101061 3934 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-r867k] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-r867k for pod on switch ip-10-0-0-29.ec2.internal I0331 03:59:15.101159 3934 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-r867k I0331 03:59:15.107961 3934 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-78d448486b-znv9f] creating logical port openshift-monitoring_metrics-server-78d448486b-znv9f for pod on switch ip-10-0-0-29.ec2.internal I0331 03:59:15.108049 3934 kube.go:256] Updating pod openshift-monitoring/metrics-server-78d448486b-znv9f I0331 03:59:15.116622 3934 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-r867k] pod update took 15.497977ms I0331 03:59:15.116645 3934 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-r867k] addLogicalPort annotation time took 15.52654ms I0331 03:59:15.117752 3934 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-r867k] addLogicalPort took 16.700605ms, libovsdb time 783.512µs I0331 03:59:15.123735 3934 pod.go:62] [openshift-monitoring/metrics-server-78d448486b-znv9f] pod update took 15.712606ms I0331 03:59:15.123754 3934 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-78d448486b-znv9f] addLogicalPort annotation time took 15.735292ms I0331 03:59:15.124836 3934 pods.go:271] [openshift-monitoring/metrics-server-78d448486b-znv9f] addLogicalPort took 16.885059ms, libovsdb time 742.678µs I0331 03:59:15.238856 3934 cni.go:403] [openshift-monitoring/alertmanager-main-1 30f66458b6c5a5019c25ea05fc24896c4e19123b0935ba4e39393861725010f0 network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 30f66458b6c5a5019c25ea05fc24896c4e19123b0935ba4e39393861725010f0 network default NAD default] I0331 03:59:15.242238 3934 cni.go:403] [openshift-monitoring/thanos-querier-57f477cc75-nwcqv 021b2deab6d1569e449bdfe52362698a85b1dc000e0ef08a122a0db7e9de0467 network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-57f477cc75-nwcqv 021b2deab6d1569e449bdfe52362698a85b1dc000e0ef08a122a0db7e9de0467 network default NAD default] I0331 03:59:15.242812 3934 cni.go:403] [openshift-monitoring/kube-state-metrics-664d8f7fc8-5qr6d a23cbdec55d7d3ee37624ef25a7b121b8e9f01db9640f749ee87d022ba639981 network default NAD default] ADD starting CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-5qr6d a23cbdec55d7d3ee37624ef25a7b121b8e9f01db9640f749ee87d022ba639981 network default NAD default] I0331 03:59:15.252806 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 30f66458b6c5a50, network: default, NAD default, SandboxID: "30f66458b6c5a5019c25ea05fc24896c4e19123b0935ba4e39393861725010f0", PCI device ID: , UID: "7c5cbaf2-2bda-4163-9226-ed40edc02db6", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0331 03:59:15.253664 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-664d8f7fc8-5qr6d, hostIfaceName: a23cbdec55d7d3e, network: default, NAD default, SandboxID: "a23cbdec55d7d3ee37624ef25a7b121b8e9f01db9640f749ee87d022ba639981", PCI device ID: , UID: "5762eb14-dc10-4435-9ba0-3b8e5c8634af", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0331 03:59:15.254549 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-57f477cc75-nwcqv, hostIfaceName: 021b2deab6d1569, network: default, NAD default, SandboxID: "021b2deab6d1569e449bdfe52362698a85b1dc000e0ef08a122a0db7e9de0467", PCI device ID: , UID: "f709d2ee-b20a-43b3-be75-74fa48d20962", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0331 03:59:15.304935 3934 cni.go:424] [openshift-monitoring/thanos-querier-57f477cc75-nwcqv 021b2deab6d1569e449bdfe52362698a85b1dc000e0ef08a122a0db7e9de0467 network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-57f477cc75-nwcqv 021b2deab6d1569e449bdfe52362698a85b1dc000e0ef08a122a0db7e9de0467 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:4c:8c:b7:b2:d0\",\"name\":\"021b2deab6d1569\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/660e38dd-6a12-4487-96a7-238f95fd4c23\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:59:15.319498 3934 cni.go:424] [openshift-monitoring/alertmanager-main-1 30f66458b6c5a5019c25ea05fc24896c4e19123b0935ba4e39393861725010f0 network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 30f66458b6c5a5019c25ea05fc24896c4e19123b0935ba4e39393861725010f0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:36:9b:10:12:aa\",\"name\":\"30f66458b6c5a50\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b2b13149-2ac5-46bc-bdf9-18235a41da83\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:59:15.338581 3934 cni.go:424] [openshift-monitoring/kube-state-metrics-664d8f7fc8-5qr6d a23cbdec55d7d3ee37624ef25a7b121b8e9f01db9640f749ee87d022ba639981 network default NAD default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-664d8f7fc8-5qr6d a23cbdec55d7d3ee37624ef25a7b121b8e9f01db9640f749ee87d022ba639981 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:4a:a2:f8:ad:e8\",\"name\":\"a23cbdec55d7d3e\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4daa12a6-6aa2-4b10-b796-f7dee6c6258e\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:59:15.480263 3934 cni.go:403] [openshift-monitoring/metrics-server-78d448486b-znv9f 9508f32de1f5aa691877e16b0bb7e5f0468e01ffede2d60adc8bf53d9c246815 network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-78d448486b-znv9f 9508f32de1f5aa691877e16b0bb7e5f0468e01ffede2d60adc8bf53d9c246815 network default NAD default] I0331 03:59:15.488836 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-78d448486b-znv9f, hostIfaceName: 9508f32de1f5aa6, network: default, NAD default, SandboxID: "9508f32de1f5aa691877e16b0bb7e5f0468e01ffede2d60adc8bf53d9c246815", PCI device ID: , UID: "fd5d6258-6cba-4780-b0e2-32d3a830f177", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0331 03:59:15.533578 3934 cni.go:424] [openshift-monitoring/metrics-server-78d448486b-znv9f 9508f32de1f5aa691877e16b0bb7e5f0468e01ffede2d60adc8bf53d9c246815 network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-78d448486b-znv9f 9508f32de1f5aa691877e16b0bb7e5f0468e01ffede2d60adc8bf53d9c246815 network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:63:3e:45:3d:25\",\"name\":\"9508f32de1f5aa6\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/94b259c9-50ad-47bf-86e2-7b785b7f2642\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 03:59:15.948842 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0331 03:59:16.631134 3934 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-6755647cfb-bb65x IP address from the namespace address-set, err: pod openshift-console/console-6755647cfb-bb65x: no pod IPs found I0331 03:59:16.828869 3934 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-0-29.ec2.internal I0331 03:59:16.828998 3934 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0331 03:59:16.855478 3934 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 26.517933ms I0331 03:59:16.855505 3934 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 26.554819ms I0331 03:59:16.856921 3934 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 28.065994ms, libovsdb time 1.030953ms I0331 03:59:18.212288 3934 obj_retry.go:464] Detected object openshift-image-registry/image-registry-848789b554-vdtgz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:59:18.212828 3934 obj_retry.go:464] Detected object openshift-image-registry/image-registry-848789b554-vdtgz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0331 03:59:18.219167 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:59:18.219171 3934 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:59:18.219226 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:59:18.219238 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:59:18.334322 3934 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:59:18.334344 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0331 03:59:18.562204 3934 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-r867k 5a64422791f084bdc77f21f8dfc7f214510cb7cd688c11ac92dd494c74eac003 network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-r867k 5a64422791f084bdc77f21f8dfc7f214510cb7cd688c11ac92dd494c74eac003 network default NAD default] I0331 03:59:18.563628 3934 cni.go:403] [openshift-monitoring/prometheus-k8s-1 a388d39da3c65f8d2f4ddf4665048e268282e576eba028e7a42420eecc41b69d network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 a388d39da3c65f8d2f4ddf4665048e268282e576eba028e7a42420eecc41b69d network default NAD default] I0331 03:59:18.573572 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-r867k, hostIfaceName: 5a64422791f084b, network: default, NAD default, SandboxID: "5a64422791f084bdc77f21f8dfc7f214510cb7cd688c11ac92dd494c74eac003", PCI device ID: , UID: "8ee55776-c1e6-4249-9468-12fcfbcb80da", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0331 03:59:18.574769 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: a388d39da3c65f8, network: default, NAD default, SandboxID: "a388d39da3c65f8d2f4ddf4665048e268282e576eba028e7a42420eecc41b69d", PCI device ID: , UID: "9a649d49-3668-43f1-8ba3-dc63a505c28d", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0331 03:59:18.622769 3934 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-r867k 5a64422791f084bdc77f21f8dfc7f214510cb7cd688c11ac92dd494c74eac003 network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-r867k 5a64422791f084bdc77f21f8dfc7f214510cb7cd688c11ac92dd494c74eac003 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:a8:a1:c6:06:97\",\"name\":\"5a64422791f084b\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2c23e659-bf4a-49da-a25d-e16c9928d8b2\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:59:18.642873 3934 cni.go:424] [openshift-monitoring/prometheus-k8s-1 a388d39da3c65f8d2f4ddf4665048e268282e576eba028e7a42420eecc41b69d network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 a388d39da3c65f8d2f4ddf4665048e268282e576eba028e7a42420eecc41b69d network default NAD default], result "{\"interfaces\":[{\"mac\":\"fa:6e:3f:85:29:46\",\"name\":\"a388d39da3c65f8\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/97914e58-1e7d-4a4d-a0e2-1ac8cf086161\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 03:59:18.944277 3934 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:18.944299 3934 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:18.944304 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:18.979834 3934 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:59:18.979855 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:59:19.001964 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0331 03:59:19.107331 3934 obj_retry.go:458] Detected object openshift-image-registry/image-registry-848789b554-vdtgz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:19.107354 3934 obj_retry.go:458] Detected object openshift-image-registry/image-registry-848789b554-vdtgz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:19.110793 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-848789b554-vdtgz I0331 03:59:19.110882 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-848789b554-vdtgz W0331 03:59:19.142047 3934 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:59:19.848743 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0331 03:59:20.306140 3934 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:20.306160 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:20.306167 3934 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:20.306173 3934 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:20.363306 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0331 03:59:20.735673 3934 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-cf44644d9-s7849 IP address from the namespace address-set, err: pod openshift-console/console-cf44644d9-s7849: no pod IPs found W0331 03:59:20.746744 3934 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-cf44644d9-6tj9j IP address from the namespace address-set, err: pod openshift-console/console-cf44644d9-6tj9j: no pod IPs found W0331 03:59:20.993243 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9443: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0331 03:59:21.080743 3934 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.080768 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.080774 3934 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.080779 3934 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.329412 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.329449 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.329457 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.329466 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.329482 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.329487 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.329491 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:21.329495 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0331 03:59:21.626352 3934 cni.go:403] [openshift-network-diagnostics/network-check-target-6b6dn d71a53790a7b252cdaa99bd6a9b945069a66dddcfd9f99e8f732fde768d86df9 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-6b6dn d71a53790a7b252cdaa99bd6a9b945069a66dddcfd9f99e8f732fde768d86df9 network default NAD default] I0331 03:59:21.636738 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-6b6dn, hostIfaceName: d71a53790a7b252, network: default, NAD default, SandboxID: "d71a53790a7b252cdaa99bd6a9b945069a66dddcfd9f99e8f732fde768d86df9", PCI device ID: , UID: "e747cf9c-d628-4c63-91c8-84f752d68615", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0331 03:59:21.646204 3934 cni.go:403] [openshift-multus/network-metrics-daemon-hjsvb 1048d32cb5872b96f88d1bf76cdb44e5f934e0b508a61ae239bc1d6bcc498958 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-hjsvb 1048d32cb5872b96f88d1bf76cdb44e5f934e0b508a61ae239bc1d6bcc498958 network default NAD default] I0331 03:59:21.657180 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-hjsvb, hostIfaceName: 1048d32cb5872b9, network: default, NAD default, SandboxID: "1048d32cb5872b96f88d1bf76cdb44e5f934e0b508a61ae239bc1d6bcc498958", PCI device ID: , UID: "41bac3e5-7c22-4048-9cc5-7db9539ed8d6", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0331 03:59:21.689702 3934 cni.go:424] [openshift-network-diagnostics/network-check-target-6b6dn d71a53790a7b252cdaa99bd6a9b945069a66dddcfd9f99e8f732fde768d86df9 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-6b6dn d71a53790a7b252cdaa99bd6a9b945069a66dddcfd9f99e8f732fde768d86df9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:02:6d:9f:24:d3\",\"name\":\"d71a53790a7b252\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4abfc927-24cc-4e5c-8551-33e14d13013f\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:59:21.711998 3934 cni.go:424] [openshift-multus/network-metrics-daemon-hjsvb 1048d32cb5872b96f88d1bf76cdb44e5f934e0b508a61ae239bc1d6bcc498958 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-hjsvb 1048d32cb5872b96f88d1bf76cdb44e5f934e0b508a61ae239bc1d6bcc498958 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:be:37:f0:20:5b\",\"name\":\"1048d32cb5872b9\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2ebb5d1e-66e5-42b2-b5a3-60d63f4b409c\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 03:59:26.991005 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:26.991026 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:26.991034 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0331 03:59:26.991045 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0331 03:59:29.007476 3934 cni.go:403] [openshift-console/console-ff4b454d6-cq7hv ebd643c5194167cd9b391df11976ad76ab10f1d35774a5d53e6a6e40f70a11c7 network default NAD default] DEL starting CNI request [openshift-console/console-ff4b454d6-cq7hv ebd643c5194167cd9b391df11976ad76ab10f1d35774a5d53e6a6e40f70a11c7 network default NAD default] I0331 03:59:29.096089 3934 cni.go:424] [openshift-console/console-ff4b454d6-cq7hv ebd643c5194167cd9b391df11976ad76ab10f1d35774a5d53e6a6e40f70a11c7 network default NAD default] DEL finished CNI request [openshift-console/console-ff4b454d6-cq7hv ebd643c5194167cd9b391df11976ad76ab10f1d35774a5d53e6a6e40f70a11c7 network default NAD default], result "{}", err I0331 03:59:29.161826 3934 obj_retry.go:464] Detected object openshift-console/console-ff4b454d6-cq7hv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:59:29.161857 3934 pods.go:176] Deleting pod: openshift-console/console-ff4b454d6-cq7hv I0331 03:59:29.163101 3934 pods.go:236] Attempting to release IPs for pod: openshift-console/console-ff4b454d6-cq7hv, ips: 10.129.0.8 I0331 03:59:29.163130 3934 obj_retry.go:464] Detected object openshift-console/console-ff4b454d6-cq7hv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:59:29.171791 3934 base_network_controller_pods.go:487] [default/openshift-console/console-5fc8b99dc7-hdppf] creating logical port openshift-console_console-5fc8b99dc7-hdppf for pod on switch ip-10-0-0-29.ec2.internal I0331 03:59:29.171876 3934 kube.go:256] Updating pod openshift-console/console-5fc8b99dc7-hdppf I0331 03:59:29.186629 3934 pod.go:62] [openshift-console/console-5fc8b99dc7-hdppf] pod update took 14.774548ms I0331 03:59:29.186655 3934 base_network_controller_pods.go:937] [default/openshift-console/console-5fc8b99dc7-hdppf] addLogicalPort annotation time took 14.803894ms I0331 03:59:29.187942 3934 pods.go:271] [openshift-console/console-5fc8b99dc7-hdppf] addLogicalPort took 16.164036ms, libovsdb time 866.784µs I0331 03:59:29.576920 3934 cni.go:403] [openshift-console/console-5fc8b99dc7-hdppf 29f9d467d139582154816a5e76484956363b7cd72436d30fe7b7e0b9c4a2aa72 network default NAD default] ADD starting CNI request [openshift-console/console-5fc8b99dc7-hdppf 29f9d467d139582154816a5e76484956363b7cd72436d30fe7b7e0b9c4a2aa72 network default NAD default] I0331 03:59:29.587733 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5fc8b99dc7-hdppf, hostIfaceName: 29f9d467d139582, network: default, NAD default, SandboxID: "29f9d467d139582154816a5e76484956363b7cd72436d30fe7b7e0b9c4a2aa72", PCI device ID: , UID: "07c25456-772f-478d-83cf-7626e06956ba", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0331 03:59:29.632319 3934 cni.go:424] [openshift-console/console-5fc8b99dc7-hdppf 29f9d467d139582154816a5e76484956363b7cd72436d30fe7b7e0b9c4a2aa72 network default NAD default] ADD finished CNI request [openshift-console/console-5fc8b99dc7-hdppf 29f9d467d139582154816a5e76484956363b7cd72436d30fe7b7e0b9c4a2aa72 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:7d:a2:ac:94:84\",\"name\":\"29f9d467d139582\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2f8e4ad1-405d-4aa9-b222-0d59b07a0965\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 03:59:30.030851 3934 obj_retry.go:458] Detected object openshift-console/console-ff4b454d6-cq7hv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:30.030882 3934 obj_retry.go:458] Detected object openshift-console/console-ff4b454d6-cq7hv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:30.040545 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-ff4b454d6-cq7hv I0331 03:59:30.040577 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-ff4b454d6-cq7hv W0331 03:59:33.400541 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:59:33.400572 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:59:33.400584 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:59:33.400589 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 03:59:38.200773 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0331 03:59:38.225228 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0331 03:59:39.964971 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0331 03:59:40.084127 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0331 03:59:40.367209 3934 obj_retry.go:464] Detected object openshift-console/console-ff4b454d6-h5kr4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:59:40.367746 3934 obj_retry.go:464] Detected object openshift-console/console-ff4b454d6-h5kr4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:59:41.193014 3934 obj_retry.go:458] Detected object openshift-console/console-ff4b454d6-h5kr4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:41.193042 3934 obj_retry.go:458] Detected object openshift-console/console-ff4b454d6-h5kr4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:41.197503 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-ff4b454d6-h5kr4 I0331 03:59:41.197530 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-ff4b454d6-h5kr4 W0331 03:59:41.893753 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:41.893773 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:41.893779 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:41.893792 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:41.893798 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:41.893805 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:41.950624 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:41.950647 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:41.950657 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0331 03:59:42.243881 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0331 03:59:42.243918 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0331 03:59:45.228300 3934 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fdbc88f75-g44js of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:59:45.228850 3934 obj_retry.go:464] Detected object openshift-image-registry/image-registry-5fdbc88f75-g44js of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:59:45.416771 3934 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fdbc88f75-g44js of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:45.416795 3934 obj_retry.go:458] Detected object openshift-image-registry/image-registry-5fdbc88f75-g44js of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:45.422165 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-5fdbc88f75-g44js I0331 03:59:45.422191 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-5fdbc88f75-g44js I0331 03:59:49.383096 3934 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mxq9x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 03:59:49.383648 3934 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mxq9x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 03:59:49.383685 3934 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mxq9x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 03:59:49.383694 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0331 03:59:49.383709 3934 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mxq9x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 03:59:49.383714 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0331 03:59:50.408209 3934 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mxq9x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:50.408238 3934 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mxq9x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:50.408253 3934 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mxq9x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:50.408266 3934 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390mxq9x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 03:59:51.111763 3934 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0331 03:59:51.120731 3934 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0331 03:59:56.275857 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0331 03:59:58.018320 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0331 04:00:04.319206 3934 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29582160-ggrdr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.319655 3934 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-29582160-ggrdr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.617920 3934 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-vt4j6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:04.618476 3934 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-vt4j6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:05.340519 3934 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29582160-ggrdr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.340545 3934 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-29582160-ggrdr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.413527 3934 obj_retry.go:464] Detected object openshift-console/console-6755647cfb-v6gx5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:05.414118 3934 obj_retry.go:464] Detected object openshift-console/console-6755647cfb-v6gx5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:05.505346 3934 obj_retry.go:458] Detected object openshift-console/console-6755647cfb-v6gx5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.505366 3934 obj_retry.go:458] Detected object openshift-console/console-6755647cfb-v6gx5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.509844 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6755647cfb-v6gx5 I0331 04:00:05.509869 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6755647cfb-v6gx5 I0331 04:00:05.519814 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-vt4j6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:05.519826 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-vt4j6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:07.392127 3934 obj_retry.go:464] Detected object openshift-ingress/router-default-5cdc57b9dc-dlhfz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:07.392653 3934 obj_retry.go:464] Detected object openshift-ingress/router-default-5cdc57b9dc-dlhfz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:08.326592 3934 obj_retry.go:458] Detected object openshift-ingress/router-default-5cdc57b9dc-dlhfz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:08.326619 3934 obj_retry.go:458] Detected object openshift-ingress/router-default-5cdc57b9dc-dlhfz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:08.331576 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5cdc57b9dc-dlhfz I0331 04:00:08.331601 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5cdc57b9dc-dlhfz W0331 04:00:17.275231 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0331 04:00:17.275252 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0331 04:00:55.402505 3934 cni.go:403] [openshift-console/console-5fc8b99dc7-hdppf 29f9d467d139582154816a5e76484956363b7cd72436d30fe7b7e0b9c4a2aa72 network default NAD default] DEL starting CNI request [openshift-console/console-5fc8b99dc7-hdppf 29f9d467d139582154816a5e76484956363b7cd72436d30fe7b7e0b9c4a2aa72 network default NAD default] I0331 04:00:55.484084 3934 cni.go:424] [openshift-console/console-5fc8b99dc7-hdppf 29f9d467d139582154816a5e76484956363b7cd72436d30fe7b7e0b9c4a2aa72 network default NAD default] DEL finished CNI request [openshift-console/console-5fc8b99dc7-hdppf 29f9d467d139582154816a5e76484956363b7cd72436d30fe7b7e0b9c4a2aa72 network default NAD default], result "{}", err I0331 04:00:55.529298 3934 obj_retry.go:464] Detected object openshift-console/console-5fc8b99dc7-hdppf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:55.529334 3934 pods.go:176] Deleting pod: openshift-console/console-5fc8b99dc7-hdppf I0331 04:00:55.530694 3934 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5fc8b99dc7-hdppf, ips: 10.129.0.21 I0331 04:00:55.530721 3934 obj_retry.go:464] Detected object openshift-console/console-5fc8b99dc7-hdppf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:00:55.538764 3934 base_network_controller_pods.go:487] [default/openshift-console/console-7b6cf9f5bd-lhfrk] creating logical port openshift-console_console-7b6cf9f5bd-lhfrk for pod on switch ip-10-0-0-29.ec2.internal I0331 04:00:55.538920 3934 kube.go:256] Updating pod openshift-console/console-7b6cf9f5bd-lhfrk I0331 04:00:55.555637 3934 pod.go:62] [openshift-console/console-7b6cf9f5bd-lhfrk] pod update took 16.767878ms I0331 04:00:55.555658 3934 base_network_controller_pods.go:937] [default/openshift-console/console-7b6cf9f5bd-lhfrk] addLogicalPort annotation time took 16.79411ms I0331 04:00:55.556924 3934 pods.go:271] [openshift-console/console-7b6cf9f5bd-lhfrk] addLogicalPort took 18.175414ms, libovsdb time 919.857µs I0331 04:00:55.894449 3934 cni.go:403] [openshift-console/console-7b6cf9f5bd-lhfrk 9a1475c9b3f0516991aa7623275e85b22840537f9f5479b39ba3a27b1f969fde network default NAD default] ADD starting CNI request [openshift-console/console-7b6cf9f5bd-lhfrk 9a1475c9b3f0516991aa7623275e85b22840537f9f5479b39ba3a27b1f969fde network default NAD default] I0331 04:00:55.903051 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-7b6cf9f5bd-lhfrk, hostIfaceName: 9a1475c9b3f0516, network: default, NAD default, SandboxID: "9a1475c9b3f0516991aa7623275e85b22840537f9f5479b39ba3a27b1f969fde", PCI device ID: , UID: "70423643-4c62-458d-a5c3-3c68f4595d94", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0331 04:00:55.948871 3934 cni.go:424] [openshift-console/console-7b6cf9f5bd-lhfrk 9a1475c9b3f0516991aa7623275e85b22840537f9f5479b39ba3a27b1f969fde network default NAD default] ADD finished CNI request [openshift-console/console-7b6cf9f5bd-lhfrk 9a1475c9b3f0516991aa7623275e85b22840537f9f5479b39ba3a27b1f969fde network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:5f:4e:c5:13:10\",\"name\":\"9a1475c9b3f0516\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd1d562c-63be-465f-ad12-ec8dc7c848a8\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:00:56.398589 3934 obj_retry.go:458] Detected object openshift-console/console-5fc8b99dc7-hdppf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:56.398620 3934 obj_retry.go:458] Detected object openshift-console/console-5fc8b99dc7-hdppf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:00:56.406065 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5fc8b99dc7-hdppf I0331 04:00:56.406087 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5fc8b99dc7-hdppf I0331 04:01:06.001371 3934 obj_retry.go:464] Detected object openshift-console/console-5fc8b99dc7-7c58p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:01:06.001947 3934 obj_retry.go:464] Detected object openshift-console/console-5fc8b99dc7-7c58p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0331 04:01:06.438300 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0331 04:01:06.583414 3934 obj_retry.go:458] Detected object openshift-console/console-5fc8b99dc7-7c58p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:06.583438 3934 obj_retry.go:458] Detected object openshift-console/console-5fc8b99dc7-7c58p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:01:06.590427 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5fc8b99dc7-7c58p I0331 04:01:06.590453 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5fc8b99dc7-7c58p I0331 04:01:49.654424 3934 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0331 04:04:24.559855 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0331 04:04:24.593762 3934 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0331 04:04:40.570001 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 W0331 04:04:55.546575 3934 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0331 04:04:59.485881 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=11 I0331 04:05:06.593905 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0331 04:05:06.593973 3934 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-xv9fb IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-xv9fb: no pod IPs found I0331 04:05:12.582394 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0331 04:05:17.559513 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0331 04:05:25.588261 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0331 04:05:26.482017 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0331 04:05:55.576714 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0331 04:06:04.231079 3934 namespace.go:98] [tsf] adding namespace I0331 04:06:04.231986 3934 namespace.go:102] [tsf] adding namespace took 882.454µs I0331 04:06:04.246693 3934 namespace.go:142] [tsf] updating namespace I0331 04:06:04.262754 3934 namespace.go:142] [tsf] updating namespace I0331 04:06:08.257826 3934 namespace.go:98] [konflux-ui] adding namespace I0331 04:06:08.258779 3934 namespace.go:102] [konflux-ui] adding namespace took 930.321µs I0331 04:06:08.261916 3934 namespace.go:98] [rhtpa-operator] adding namespace I0331 04:06:08.262059 3934 namespace.go:98] [cert-manager-operator] adding namespace I0331 04:06:08.262192 3934 namespace.go:98] [tsf-tpa] adding namespace I0331 04:06:08.262205 3934 namespace.go:98] [tsf-tas] adding namespace I0331 04:06:08.262600 3934 namespace.go:102] [rhtpa-operator] adding namespace took 667.768µs I0331 04:06:08.263207 3934 namespace.go:102] [cert-manager-operator] adding namespace took 1.135907ms I0331 04:06:08.263831 3934 namespace.go:102] [tsf-tpa] adding namespace took 1.62527ms I0331 04:06:08.264502 3934 namespace.go:102] [tsf-tas] adding namespace took 2.284737ms I0331 04:06:08.269186 3934 namespace.go:98] [rhbk-operator] adding namespace I0331 04:06:08.269196 3934 namespace.go:98] [tsf-keycloak] adding namespace I0331 04:06:08.269916 3934 namespace.go:102] [rhbk-operator] adding namespace took 715.994µs I0331 04:06:08.270412 3934 namespace.go:102] [tsf-keycloak] adding namespace took 1.170179ms I0331 04:06:08.270432 3934 namespace.go:98] [konflux-operator] adding namespace I0331 04:06:08.270877 3934 namespace.go:102] [konflux-operator] adding namespace took 437.19µs I0331 04:06:08.276092 3934 namespace.go:142] [konflux-ui] updating namespace I0331 04:06:08.288279 3934 namespace.go:142] [konflux-ui] updating namespace I0331 04:06:08.295207 3934 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:06:08.312357 3934 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:06:08.321094 3934 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:06:08.332864 3934 namespace.go:142] [tsf-tas] updating namespace I0331 04:06:08.332983 3934 namespace.go:142] [tsf-tpa] updating namespace I0331 04:06:08.333057 3934 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:06:08.333137 3934 namespace.go:142] [rhbk-operator] updating namespace I0331 04:06:08.333260 3934 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:06:08.340415 3934 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:06:08.345335 3934 namespace.go:142] [konflux-operator] updating namespace I0331 04:06:08.345426 3934 namespace.go:142] [konflux-ui] updating namespace I0331 04:06:08.379268 3934 namespace.go:142] [tsf-tas] updating namespace I0331 04:06:08.394346 3934 namespace.go:142] [tsf-tas] updating namespace I0331 04:06:08.394392 3934 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:06:08.407312 3934 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:06:08.415675 3934 namespace.go:142] [rhbk-operator] updating namespace I0331 04:06:08.443423 3934 namespace.go:142] [rhbk-operator] updating namespace I0331 04:06:08.443497 3934 namespace.go:142] [konflux-operator] updating namespace I0331 04:06:08.446284 3934 namespace.go:142] [tsf-tpa] updating namespace I0331 04:06:08.452176 3934 namespace.go:142] [konflux-operator] updating namespace I0331 04:06:08.463203 3934 namespace.go:142] [tsf-tpa] updating namespace I0331 04:06:08.465700 3934 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:06:09.351144 3934 namespace.go:142] [tsf-tpa] updating namespace I0331 04:06:09.361052 3934 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:06:15.007179 3934 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1x2qdb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:15.007677 3934 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1x2qdb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:15.007704 3934 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1x2qdb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:15.007713 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:06:15.007728 3934 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1x2qdb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:15.007736 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:06:15.038256 3934 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bft8kgv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:15.038671 3934 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bft8kgv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:15.038690 3934 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bft8kgv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:15.038697 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:06:15.038710 3934 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bft8kgv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:15.038714 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:06:16.033205 3934 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1x2qdb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:16.033233 3934 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1x2qdb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:16.033246 3934 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1x2qdb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:16.033264 3934 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d1x2qdb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:16.064790 3934 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bft8kgv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:16.064810 3934 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bft8kgv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:16.064826 3934 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bft8kgv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:16.064841 3934 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bft8kgv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:20.069568 3934 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba899t6x of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:20.070166 3934 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba899t6x of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:20.070193 3934 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba899t6x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:20.070205 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:06:20.070226 3934 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba899t6x of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:20.070230 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:06:20.091322 3934 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e59p2p4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:20.091700 3934 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e59p2p4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:20.091719 3934 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e59p2p4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:20.091726 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:06:20.091740 3934 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e59p2p4 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:20.091745 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:06:21.098790 3934 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba899t6x of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:21.098818 3934 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba899t6x of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:21.098831 3934 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba899t6x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:21.098844 3934 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba899t6x of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:21.123655 3934 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e59p2p4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:21.123679 3934 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e59p2p4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:21.123691 3934 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e59p2p4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:21.123714 3934 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e59p2p4 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:21.599631 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0331 04:06:22.083657 3934 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7xxfrv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:22.084219 3934 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7xxfrv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:22.084252 3934 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7xxfrv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:22.084262 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:06:22.084276 3934 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7xxfrv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:22.084285 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:06:22.116276 3934 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf67pwn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:22.116693 3934 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf67pwn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:22.116717 3934 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf67pwn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:22.116725 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0331 04:06:22.116743 3934 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf67pwn of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0331 04:06:22.116753 3934 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0331 04:06:23.123444 3934 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7xxfrv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:23.123465 3934 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7xxfrv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:23.123475 3934 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7xxfrv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:23.123489 3934 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7xxfrv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:23.148398 3934 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf67pwn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:23.148419 3934 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf67pwn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:23.148432 3934 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf67pwn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:23.148443 3934 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf67pwn of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:33.089748 3934 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-qzmdh] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-qzmdh for pod on switch ip-10-0-0-29.ec2.internal I0331 04:06:33.089948 3934 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-qzmdh I0331 04:06:33.108193 3934 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-qzmdh] pod update took 18.273741ms I0331 04:06:33.108214 3934 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-qzmdh] addLogicalPort annotation time took 18.320521ms I0331 04:06:33.109365 3934 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-qzmdh] addLogicalPort took 19.633998ms, libovsdb time 851.207µs I0331 04:06:33.445853 3934 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-qzmdh 4c8db4ca0292b568fe55a6b7c1fef36609e8dde968649628c2e24751682ce348 network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-qzmdh 4c8db4ca0292b568fe55a6b7c1fef36609e8dde968649628c2e24751682ce348 network default NAD default] I0331 04:06:33.454307 3934 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-qzmdh, hostIfaceName: 4c8db4ca0292b56, network: default, NAD default, SandboxID: "4c8db4ca0292b568fe55a6b7c1fef36609e8dde968649628c2e24751682ce348", PCI device ID: , UID: "258001b9-dca8-416a-a6e1-d29e8c29b4a2", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0331 04:06:33.498084 3934 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-qzmdh 4c8db4ca0292b568fe55a6b7c1fef36609e8dde968649628c2e24751682ce348 network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-qzmdh 4c8db4ca0292b568fe55a6b7c1fef36609e8dde968649628c2e24751682ce348 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:a7:21:c4:20:0b\",\"name\":\"4c8db4ca0292b56\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8a6082c2-31ee-4293-ae6a-8dde6b4931bc\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:06:33.607127 3934 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-dstmk for pod on switch ip-10-0-0-29.ec2.internal I0331 04:06:33.607276 3934 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk I0331 04:06:33.628384 3934 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk] pod update took 21.132123ms I0331 04:06:33.628404 3934 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk] addLogicalPort annotation time took 21.15713ms I0331 04:06:33.629553 3934 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk] addLogicalPort took 22.439467ms, libovsdb time 820.817µs I0331 04:06:33.964019 3934 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk f6cdae1592f6baf524a45147e442a44bfeb9cd8bb671d567575a463d1eedf961 network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk f6cdae1592f6baf524a45147e442a44bfeb9cd8bb671d567575a463d1eedf961 network default NAD default] I0331 04:06:33.972789 3934 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-dstmk, hostIfaceName: f6cdae1592f6baf, network: default, NAD default, SandboxID: "f6cdae1592f6baf524a45147e442a44bfeb9cd8bb671d567575a463d1eedf961", PCI device ID: , UID: "b0fbf99d-00c2-4ad2-87f7-bc278110077a", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0331 04:06:34.016508 3934 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk f6cdae1592f6baf524a45147e442a44bfeb9cd8bb671d567575a463d1eedf961 network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk f6cdae1592f6baf524a45147e442a44bfeb9cd8bb671d567575a463d1eedf961 network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:e9:fe:de:35:9f\",\"name\":\"f6cdae1592f6baf\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/df0e7b2f-7679-4ac1-aad0-2766411d5a51\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:06:35.094447 3934 namespace.go:142] [openshift-operators] updating namespace I0331 04:06:35.103385 3934 namespace.go:142] [openshift-operators] updating namespace W0331 04:06:37.811351 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0331 04:06:38.983913 3934 namespace.go:98] [cert-manager] adding namespace I0331 04:06:38.984725 3934 namespace.go:102] [cert-manager] adding namespace took 787.862µs I0331 04:06:39.034643 3934 namespace.go:142] [cert-manager] updating namespace I0331 04:06:39.069427 3934 namespace.go:142] [cert-manager] updating namespace W0331 04:06:41.086322 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:06:43.129459 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:06:43.145329 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:06:43.151339 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0331 04:06:43.161442 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0331 04:06:43.932515 3934 namespace.go:142] [openshift-operators] updating namespace W0331 04:06:44.147763 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0331 04:06:44.158034 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0331 04:06:44.466034 3934 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-pf94d] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-pf94d for pod on switch ip-10-0-0-29.ec2.internal I0331 04:06:44.466149 3934 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-pf94d I0331 04:06:44.513477 3934 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-pf94d] pod update took 47.352466ms I0331 04:06:44.513500 3934 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-pf94d] addLogicalPort annotation time took 47.380613ms I0331 04:06:44.514673 3934 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-pf94d] addLogicalPort took 48.660565ms, libovsdb time 826.491µs I0331 04:06:44.534356 3934 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-qgjhm] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-qgjhm for pod on switch ip-10-0-0-29.ec2.internal I0331 04:06:44.534490 3934 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-qgjhm I0331 04:06:44.580305 3934 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-qgjhm] pod update took 45.835968ms I0331 04:06:44.580326 3934 base_network_controller_pods.go:937] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-qgjhm] addLogicalPort annotation time took 45.86131ms I0331 04:06:44.581542 3934 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-qgjhm] addLogicalPort took 47.214273ms, libovsdb time 882.222µs I0331 04:06:44.831446 3934 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-pf94d c14ffa08d08b86fabeb87e1baf6ab61ed24d82f52d20233f01535bd60c011426 network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-pf94d c14ffa08d08b86fabeb87e1baf6ab61ed24d82f52d20233f01535bd60c011426 network default NAD default] I0331 04:06:44.840088 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-pf94d, hostIfaceName: c14ffa08d08b86f, network: default, NAD default, SandboxID: "c14ffa08d08b86fabeb87e1baf6ab61ed24d82f52d20233f01535bd60c011426", PCI device ID: , UID: "8d09f259-48dd-47c5-8222-03c4a036f082", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0331 04:06:44.876573 3934 cni.go:403] [openshift-operators/tekton-operator-webhook-668b6fdd5d-qgjhm e52550f0274e65d44db87cb07ca856e7d501aaf22b1a67ec06d75d786c95d1f4 network default NAD default] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-qgjhm e52550f0274e65d44db87cb07ca856e7d501aaf22b1a67ec06d75d786c95d1f4 network default NAD default] I0331 04:06:44.885750 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-qgjhm, hostIfaceName: e52550f0274e65d, network: default, NAD default, SandboxID: "e52550f0274e65d44db87cb07ca856e7d501aaf22b1a67ec06d75d786c95d1f4", PCI device ID: , UID: "8c860193-be64-4d34-8283-b75ba49c6141", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0331 04:06:44.886731 3934 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-pf94d c14ffa08d08b86fabeb87e1baf6ab61ed24d82f52d20233f01535bd60c011426 network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-pf94d c14ffa08d08b86fabeb87e1baf6ab61ed24d82f52d20233f01535bd60c011426 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:2e:71:cf:ea:0f\",\"name\":\"c14ffa08d08b86f\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6e60ec1c-0265-4487-a2f2-95c35edfb028\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:06:44.931208 3934 cni.go:424] [openshift-operators/tekton-operator-webhook-668b6fdd5d-qgjhm e52550f0274e65d44db87cb07ca856e7d501aaf22b1a67ec06d75d786c95d1f4 network default NAD default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-qgjhm e52550f0274e65d44db87cb07ca856e7d501aaf22b1a67ec06d75d786c95d1f4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:ca:86:bd:04:eb\",\"name\":\"e52550f0274e65d\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb6bdd04-25c3-4985-8f45-2735d47ffe97\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 04:06:45.134343 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0331 04:06:45.134362 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0331 04:06:48.770327 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0331 04:06:53.307343 3934 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:53.307844 3934 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:06:53.406358 3934 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:53.406381 3934 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:06:53.449035 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0331 04:06:53.449058 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0331 04:06:55.025669 3934 namespace.go:98] [trusted-artifact-signer] adding namespace I0331 04:06:55.026762 3934 namespace.go:102] [trusted-artifact-signer] adding namespace took 1.07006ms I0331 04:06:55.044729 3934 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:06:55.062546 3934 namespace.go:142] [trusted-artifact-signer] updating namespace W0331 04:06:55.206343 3934 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0331 04:06:59.632023 3934 namespace.go:98] [openshift-pipelines] adding namespace I0331 04:06:59.632881 3934 namespace.go:102] [openshift-pipelines] adding namespace took 835.487µs I0331 04:06:59.655423 3934 namespace.go:142] [openshift-pipelines] updating namespace I0331 04:07:00.346929 3934 namespace.go:142] [cert-manager] updating namespace I0331 04:07:00.556002 3934 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:07:01.093080 3934 namespace.go:142] [konflux-operator] updating namespace I0331 04:07:01.322338 3934 namespace.go:142] [konflux-ui] updating namespace I0331 04:07:01.544414 3934 namespace.go:142] [open-cluster-management-2pcf6lkp4abd3rm3r9e30v12cg53upbr] updating namespace I0331 04:07:01.945863 3934 namespace.go:142] [rhbk-operator] updating namespace I0331 04:07:02.148434 3934 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:07:02.355974 3934 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:07:02.547760 3934 namespace.go:142] [tsf] updating namespace I0331 04:07:02.748981 3934 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:07:02.953422 3934 namespace.go:142] [tsf-tas] updating namespace I0331 04:07:03.151274 3934 namespace.go:142] [tsf-tpa] updating namespace I0331 04:07:03.279463 3934 namespace.go:142] [cert-manager] updating namespace I0331 04:07:03.289104 3934 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:07:03.296038 3934 namespace.go:142] [default] updating namespace I0331 04:07:03.305979 3934 namespace.go:142] [konflux-operator] updating namespace I0331 04:07:03.315804 3934 namespace.go:142] [konflux-ui] updating namespace I0331 04:07:03.323157 3934 namespace.go:142] [open-cluster-management-2pcf6lkp4abd3rm3r9e30v12cg53upbr] updating namespace I0331 04:07:03.331002 3934 namespace.go:142] [openshift] updating namespace I0331 04:07:03.338164 3934 namespace.go:142] [rhbk-operator] updating namespace I0331 04:07:03.347590 3934 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:07:03.354933 3934 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:07:03.369522 3934 namespace.go:142] [tsf] updating namespace I0331 04:07:03.391038 3934 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:07:03.412241 3934 namespace.go:142] [tsf-tas] updating namespace I0331 04:07:03.431847 3934 namespace.go:142] [tsf-tpa] updating namespace I0331 04:07:03.531712 3934 namespace.go:142] [cert-manager] updating namespace I0331 04:07:03.634056 3934 namespace.go:142] [cert-manager-operator] updating namespace I0331 04:07:03.738722 3934 namespace.go:142] [default] updating namespace I0331 04:07:03.844665 3934 namespace.go:142] [konflux-operator] updating namespace I0331 04:07:03.934559 3934 namespace.go:142] [konflux-ui] updating namespace I0331 04:07:04.031995 3934 namespace.go:142] [open-cluster-management-2pcf6lkp4abd3rm3r9e30v12cg53upbr] updating namespace I0331 04:07:04.132312 3934 namespace.go:142] [openshift] updating namespace I0331 04:07:04.231101 3934 namespace.go:142] [rhbk-operator] updating namespace W0331 04:07:04.268650 3934 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0331 04:07:04.330249 3934 namespace.go:142] [rhtpa-operator] updating namespace I0331 04:07:04.429508 3934 namespace.go:142] [trusted-artifact-signer] updating namespace I0331 04:07:04.530964 3934 namespace.go:142] [tsf] updating namespace I0331 04:07:04.632662 3934 namespace.go:142] [tsf-keycloak] updating namespace I0331 04:07:04.732776 3934 namespace.go:142] [tsf-tas] updating namespace I0331 04:07:04.834481 3934 namespace.go:142] [tsf-tpa] updating namespace I0331 04:07:05.108828 3934 namespace.go:142] [openshift-pipelines] updating namespace W0331 04:07:05.256011 3934 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0331 04:07:12.922980 3934 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-b79bdd59-x5fhr] creating logical port openshift-pipelines_tekton-events-controller-b79bdd59-x5fhr for pod on switch ip-10-0-0-29.ec2.internal I0331 04:07:12.923073 3934 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-b79bdd59-x5fhr I0331 04:07:12.941303 3934 pod.go:62] [openshift-pipelines/tekton-events-controller-b79bdd59-x5fhr] pod update took 18.251994ms I0331 04:07:12.941343 3934 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-events-controller-b79bdd59-x5fhr] addLogicalPort annotation time took 18.295703ms I0331 04:07:12.942749 3934 pods.go:271] [openshift-pipelines/tekton-events-controller-b79bdd59-x5fhr] addLogicalPort took 19.786117ms, libovsdb time 1.026513ms I0331 04:07:12.992323 3934 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-28g25] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-28g25 for pod on switch ip-10-0-0-29.ec2.internal I0331 04:07:12.992404 3934 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-28g25 I0331 04:07:13.008378 3934 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-28g25] pod update took 15.996073ms I0331 04:07:13.008404 3934 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-28g25] addLogicalPort annotation time took 16.025489ms I0331 04:07:13.009558 3934 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-28g25] addLogicalPort took 17.253662ms, libovsdb time 800.041µs I0331 04:07:13.104408 3934 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-bg2sp] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-bg2sp for pod on switch ip-10-0-0-29.ec2.internal I0331 04:07:13.104619 3934 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-bg2sp I0331 04:07:13.125810 3934 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-bg2sp] pod update took 21.28226ms I0331 04:07:13.125831 3934 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-bg2sp] addLogicalPort annotation time took 21.30979ms I0331 04:07:13.127408 3934 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-bg2sp] addLogicalPort took 23.019986ms, libovsdb time 1.085165ms I0331 04:07:13.299536 3934 cni.go:403] [openshift-pipelines/tekton-events-controller-b79bdd59-x5fhr 4ed16c91ff9fb92965dcfd798d1bac126c3484289d40909e6b1629bfafc92d39 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-x5fhr 4ed16c91ff9fb92965dcfd798d1bac126c3484289d40909e6b1629bfafc92d39 network default NAD default] I0331 04:07:13.308933 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-b79bdd59-x5fhr, hostIfaceName: 4ed16c91ff9fb92, network: default, NAD default, SandboxID: "4ed16c91ff9fb92965dcfd798d1bac126c3484289d40909e6b1629bfafc92d39", PCI device ID: , UID: "e8b41e2a-71ee-457a-98cd-0536896e9257", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0331 04:07:13.355866 3934 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-28g25 76d02cf5bbac1b112c5a82a97cf2d9902a39d95fdd95e9047e6996eff8040165 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-28g25 76d02cf5bbac1b112c5a82a97cf2d9902a39d95fdd95e9047e6996eff8040165 network default NAD default] I0331 04:07:13.358282 3934 cni.go:424] [openshift-pipelines/tekton-events-controller-b79bdd59-x5fhr 4ed16c91ff9fb92965dcfd798d1bac126c3484289d40909e6b1629bfafc92d39 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-x5fhr 4ed16c91ff9fb92965dcfd798d1bac126c3484289d40909e6b1629bfafc92d39 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:7c:92:b7:bc:71\",\"name\":\"4ed16c91ff9fb92\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/68b49c3b-2a16-41ec-9c2b-c88119f7888a\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:07:13.365692 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-28g25, hostIfaceName: 76d02cf5bbac1b1, network: default, NAD default, SandboxID: "76d02cf5bbac1b112c5a82a97cf2d9902a39d95fdd95e9047e6996eff8040165", PCI device ID: , UID: "0d215be6-e458-4ae2-9ec7-6445b1d2e582", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0331 04:07:13.412384 3934 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-28g25 76d02cf5bbac1b112c5a82a97cf2d9902a39d95fdd95e9047e6996eff8040165 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-28g25 76d02cf5bbac1b112c5a82a97cf2d9902a39d95fdd95e9047e6996eff8040165 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:70:0f:38:2f:51\",\"name\":\"76d02cf5bbac1b1\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/49f96982-fbf5-419d-81a7-31aed9b7fedb\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:07:13.470271 3934 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-bg2sp 8f83e5dc96effde85da5c3cafeeaf13043607e58e2a3ce84fbf4a4cd25c39ed0 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-bg2sp 8f83e5dc96effde85da5c3cafeeaf13043607e58e2a3ce84fbf4a4cd25c39ed0 network default NAD default] I0331 04:07:13.480831 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-bg2sp, hostIfaceName: 8f83e5dc96effde, network: default, NAD default, SandboxID: "8f83e5dc96effde85da5c3cafeeaf13043607e58e2a3ce84fbf4a4cd25c39ed0", PCI device ID: , UID: "55429a34-1830-4a54-aecb-39b905b1975e", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0331 04:07:13.527009 3934 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-bg2sp 8f83e5dc96effde85da5c3cafeeaf13043607e58e2a3ce84fbf4a4cd25c39ed0 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-bg2sp 8f83e5dc96effde85da5c3cafeeaf13043607e58e2a3ce84fbf4a4cd25c39ed0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:99:c4:91:a1:ea\",\"name\":\"8f83e5dc96effde\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cc3cb311-5999-4243-ae6d-ba7b2a69a9ca\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 04:07:16.935369 3934 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:07:16.935387 3934 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:07:16.935392 3934 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:07:16.935396 3934 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:07:16.967867 3934 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:07:16.967887 3934 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:07:16.967906 3934 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:07:17.268246 3934 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:07:17.268267 3934 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:07:17.268272 3934 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0331 04:07:27.935659 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:07:27.935680 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:07:27.935685 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0331 04:07:27.959835 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:07:27.959856 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:07:27.959862 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0331 04:07:27.959867 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0331 04:07:29.484652 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=63 I0331 04:07:30.490472 3934 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:30.491061 3934 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:30.513207 3934 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:30.513228 3934 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:30.519173 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0331 04:07:30.519196 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure W0331 04:07:37.418434 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0331 04:07:37.418517 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:07:37.418531 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:07:39.447977 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0331 04:07:47.485150 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=163 I0331 04:07:49.565653 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0331 04:07:51.496381 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=638 I0331 04:07:52.325503 3934 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-v69sz] creating logical port openshift-pipelines_tekton-chains-controller-5fb999ffff-v69sz for pod on switch ip-10-0-0-29.ec2.internal I0331 04:07:52.325594 3934 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-5fb999ffff-v69sz I0331 04:07:52.341568 3934 pod.go:62] [openshift-pipelines/tekton-chains-controller-5fb999ffff-v69sz] pod update took 15.999029ms I0331 04:07:52.341591 3934 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-v69sz] addLogicalPort annotation time took 16.026541ms I0331 04:07:52.342934 3934 pods.go:271] [openshift-pipelines/tekton-chains-controller-5fb999ffff-v69sz] addLogicalPort took 17.444683ms, libovsdb time 966.738µs I0331 04:07:52.699282 3934 cni.go:403] [openshift-pipelines/tekton-chains-controller-5fb999ffff-v69sz 713e3b1f572bb6dd57e44bf506eb32ee36cb13a98ab606c678bb4cc9926dcd07 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-v69sz 713e3b1f572bb6dd57e44bf506eb32ee36cb13a98ab606c678bb4cc9926dcd07 network default NAD default] I0331 04:07:52.708077 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-5fb999ffff-v69sz, hostIfaceName: 713e3b1f572bb6d, network: default, NAD default, SandboxID: "713e3b1f572bb6dd57e44bf506eb32ee36cb13a98ab606c678bb4cc9926dcd07", PCI device ID: , UID: "9966b700-dcac-417c-89f1-f13c17c4d0af", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0331 04:07:52.754999 3934 cni.go:424] [openshift-pipelines/tekton-chains-controller-5fb999ffff-v69sz 713e3b1f572bb6dd57e44bf506eb32ee36cb13a98ab606c678bb4cc9926dcd07 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-v69sz 713e3b1f572bb6dd57e44bf506eb32ee36cb13a98ab606c678bb4cc9926dcd07 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:b7:97:65:fd:4b\",\"name\":\"713e3b1f572bb6d\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e585c5c8-816a-447c-a7d8-6dfa63a4952f\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:07:55.734858 3934 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:55.735443 3934 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:07:56.515506 3934 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:56.515532 3934 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:07:56.523382 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0331 04:07:56.523407 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0331 04:07:58.524670 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:07:58.524691 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:07:58.524724 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service I0331 04:07:59.185494 3934 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0331 04:08:02.284106 3934 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-0-29.ec2.internal I0331 04:08:02.284194 3934 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0331 04:08:02.297070 3934 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 12.895528ms I0331 04:08:02.297098 3934 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 12.928661ms I0331 04:08:02.298385 3934 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 14.296938ms, libovsdb time 884.103µs I0331 04:08:05.216130 3934 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-235.ec2.internal I0331 04:08:05.216154 3934 egressfirewall.go:628] Syncing node "ip-10-0-2-235.ec2.internal" for egress firewall I0331 04:08:05.216167 3934 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-235.ec2.internal: 45.697µs I0331 04:08:08.343176 3934 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 0d395e95c57c1e98169834f9798d6313417dfbb5ddb7a52ed092e679bf50ea43 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 0d395e95c57c1e98169834f9798d6313417dfbb5ddb7a52ed092e679bf50ea43 network default NAD default] I0331 04:08:08.352092 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 0d395e95c57c1e9, network: default, NAD default, SandboxID: "0d395e95c57c1e98169834f9798d6313417dfbb5ddb7a52ed092e679bf50ea43", PCI device ID: , UID: "b4526a96-01ff-4420-8e94-dcd791c82cae", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0331 04:08:08.398796 3934 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 0d395e95c57c1e98169834f9798d6313417dfbb5ddb7a52ed092e679bf50ea43 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 0d395e95c57c1e98169834f9798d6313417dfbb5ddb7a52ed092e679bf50ea43 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:ec:93:0a:f7:ea\",\"name\":\"0d395e95c57c1e9\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e80b518e-c989-4118-9177-c05d5978347e\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:08:09.599504 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 I0331 04:08:14.256777 3934 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0331 04:08:15.594516 3934 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-kdpvq] creating logical port openshift-pipelines_tekton-results-api-7b4b7bb78c-kdpvq for pod on switch ip-10-0-0-29.ec2.internal I0331 04:08:15.594639 3934 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-7b4b7bb78c-kdpvq I0331 04:08:15.615800 3934 pod.go:62] [openshift-pipelines/tekton-results-api-7b4b7bb78c-kdpvq] pod update took 21.198726ms I0331 04:08:15.615825 3934 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-api-7b4b7bb78c-kdpvq] addLogicalPort annotation time took 21.231037ms I0331 04:08:15.617254 3934 pods.go:271] [openshift-pipelines/tekton-results-api-7b4b7bb78c-kdpvq] addLogicalPort took 22.758558ms, libovsdb time 941.046µs I0331 04:08:15.689406 3934 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-cgtrp] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-55cbb85cb8-cgtrp for pod on switch ip-10-0-0-29.ec2.internal I0331 04:08:15.689509 3934 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-cgtrp I0331 04:08:15.702755 3934 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-cgtrp] pod update took 13.268678ms I0331 04:08:15.702776 3934 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-cgtrp] addLogicalPort annotation time took 13.295867ms I0331 04:08:15.704036 3934 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-cgtrp] addLogicalPort took 14.648616ms, libovsdb time 849.986µs I0331 04:08:15.956271 3934 cni.go:403] [openshift-pipelines/tekton-results-api-7b4b7bb78c-kdpvq 94f67a06fe8a36d16ed1ec53e7d34ab7f14672e42dd8f89cf40b611975e40df8 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-kdpvq 94f67a06fe8a36d16ed1ec53e7d34ab7f14672e42dd8f89cf40b611975e40df8 network default NAD default] I0331 04:08:15.964923 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-7b4b7bb78c-kdpvq, hostIfaceName: 94f67a06fe8a36d, network: default, NAD default, SandboxID: "94f67a06fe8a36d16ed1ec53e7d34ab7f14672e42dd8f89cf40b611975e40df8", PCI device ID: , UID: "52eec6a3-edf1-469a-8494-e0d4971dd12e", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0331 04:08:16.011695 3934 cni.go:424] [openshift-pipelines/tekton-results-api-7b4b7bb78c-kdpvq 94f67a06fe8a36d16ed1ec53e7d34ab7f14672e42dd8f89cf40b611975e40df8 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-api-7b4b7bb78c-kdpvq 94f67a06fe8a36d16ed1ec53e7d34ab7f14672e42dd8f89cf40b611975e40df8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:c5:b8:0a:74:c4\",\"name\":\"94f67a06fe8a36d\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c9867552-809a-47a8-a547-24fc0e99bf14\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:08:16.048869 3934 cni.go:403] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-cgtrp 885f68d00f749db6bdb488377fef3bf4e9956e256c14aef62ec9db6ede396f50 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-cgtrp 885f68d00f749db6bdb488377fef3bf4e9956e256c14aef62ec9db6ede396f50 network default NAD default] I0331 04:08:16.057228 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-55cbb85cb8-cgtrp, hostIfaceName: 885f68d00f749db, network: default, NAD default, SandboxID: "885f68d00f749db6bdb488377fef3bf4e9956e256c14aef62ec9db6ede396f50", PCI device ID: , UID: "4f03ee1f-3e00-4b93-bdac-4f2e11da9ca3", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0331 04:08:16.102671 3934 cni.go:424] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-cgtrp 885f68d00f749db6bdb488377fef3bf4e9956e256c14aef62ec9db6ede396f50 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-cgtrp 885f68d00f749db6bdb488377fef3bf4e9956e256c14aef62ec9db6ede396f50 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:94:f1:ec:25:8f\",\"name\":\"885f68d00f749db\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c460b50b-f014-43d2-802b-31c144ae374d\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 04:08:20.259326 3934 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:08:20.259347 3934 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:08:20.259353 3934 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0331 04:08:21.483965 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=88 I0331 04:08:25.835838 3934 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:25.836380 3934 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:25.880177 3934 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:25.880196 3934 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:25.886429 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0331 04:08:25.886450 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam W0331 04:08:26.289071 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:08:26.289092 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0331 04:08:26.289097 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0331 04:08:28.619730 3934 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-799c6898b8-dnqbt] creating logical port tsf-tas_fulcio-server-799c6898b8-dnqbt for pod on switch ip-10-0-0-29.ec2.internal I0331 04:08:28.619817 3934 kube.go:256] Updating pod tsf-tas/fulcio-server-799c6898b8-dnqbt I0331 04:08:28.668292 3934 pod.go:62] [tsf-tas/fulcio-server-799c6898b8-dnqbt] pod update took 48.493619ms I0331 04:08:28.668314 3934 base_network_controller_pods.go:937] [default/tsf-tas/fulcio-server-799c6898b8-dnqbt] addLogicalPort annotation time took 48.529796ms I0331 04:08:28.669522 3934 pods.go:271] [tsf-tas/fulcio-server-799c6898b8-dnqbt] addLogicalPort took 49.808287ms, libovsdb time 796.177µs I0331 04:08:28.837559 3934 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-6lgbn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:28.838117 3934 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-6lgbn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:29.002310 3934 cni.go:403] [tsf-tas/fulcio-server-799c6898b8-dnqbt ddcd3bcaf89402abbc9757093f6c76855db5105b318a09db4e2759d581965064 network default NAD default] ADD starting CNI request [tsf-tas/fulcio-server-799c6898b8-dnqbt ddcd3bcaf89402abbc9757093f6c76855db5105b318a09db4e2759d581965064 network default NAD default] I0331 04:08:29.019964 3934 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-799c6898b8-dnqbt, hostIfaceName: ddcd3bcaf89402a, network: default, NAD default, SandboxID: "ddcd3bcaf89402abbc9757093f6c76855db5105b318a09db4e2759d581965064", PCI device ID: , UID: "23105b8f-25f2-430f-b128-4dfd71f70271", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0331 04:08:29.071272 3934 cni.go:424] [tsf-tas/fulcio-server-799c6898b8-dnqbt ddcd3bcaf89402abbc9757093f6c76855db5105b318a09db4e2759d581965064 network default NAD default] ADD finished CNI request [tsf-tas/fulcio-server-799c6898b8-dnqbt ddcd3bcaf89402abbc9757093f6c76855db5105b318a09db4e2759d581965064 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:4e:5f:7e:fd:d2\",\"name\":\"ddcd3bcaf89402a\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5a114df2-74bd-4a95-bfca-8274c2c7e9b1\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:08:29.859973 3934 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-6lgbn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:29.860032 3934 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-6lgbn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:30.210964 3934 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:30.211685 3934 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:30.722567 3934 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:30.722598 3934 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:30.725429 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0331 04:08:30.725457 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0331 04:08:31.310904 3934 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:08:31.310929 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:08:33.752749 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0331 04:08:33.756192 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:08:33.756209 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0331 04:08:36.775769 3934 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0331 04:08:36.804705 3934 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0331 04:08:40.866609 3934 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-mgxtp-bsrtn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:40.867219 3934 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-mgxtp-bsrtn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:08:41.744117 3934 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-mgxtp-bsrtn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:08:41.744144 3934 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-mgxtp-bsrtn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0331 04:08:42.323405 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:08:42.323424 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server W0331 04:08:44.754944 3934 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0331 04:09:14.096202 3934 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-pkr9n-zh8k7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:09:14.096793 3934 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-pkr9n-zh8k7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:09:14.990550 3934 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-pkr9n-zh8k7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:09:14.990577 3934 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-pkr9n-zh8k7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0331 04:09:22.964390 3934 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0331 04:09:23.182615 3934 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-7bcbc45bc6-ng8ft] creating logical port tsf-tas_rekor-server-7bcbc45bc6-ng8ft for pod on switch ip-10-0-0-29.ec2.internal I0331 04:09:23.182716 3934 kube.go:256] Updating pod tsf-tas/rekor-server-7bcbc45bc6-ng8ft I0331 04:09:23.195877 3934 pod.go:62] [tsf-tas/rekor-server-7bcbc45bc6-ng8ft] pod update took 13.18223ms I0331 04:09:23.195921 3934 base_network_controller_pods.go:937] [default/tsf-tas/rekor-server-7bcbc45bc6-ng8ft] addLogicalPort annotation time took 13.232722ms I0331 04:09:23.197401 3934 pods.go:271] [tsf-tas/rekor-server-7bcbc45bc6-ng8ft] addLogicalPort took 14.809704ms, libovsdb time 964.651µs W0331 04:09:24.030020 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0331 04:09:29.842795 3934 cni.go:403] [tsf-tas/rekor-server-7bcbc45bc6-ng8ft 3eff5e8d4a69201a430b1ee0cb00cec76dcc59e8002515faae77967b6cae54cc network default NAD default] ADD starting CNI request [tsf-tas/rekor-server-7bcbc45bc6-ng8ft 3eff5e8d4a69201a430b1ee0cb00cec76dcc59e8002515faae77967b6cae54cc network default NAD default] I0331 04:09:29.852161 3934 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-7bcbc45bc6-ng8ft, hostIfaceName: 3eff5e8d4a69201, network: default, NAD default, SandboxID: "3eff5e8d4a69201a430b1ee0cb00cec76dcc59e8002515faae77967b6cae54cc", PCI device ID: , UID: "ede5125d-9ab0-49c6-8bea-5fae79a55fcf", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0331 04:09:29.899650 3934 cni.go:424] [tsf-tas/rekor-server-7bcbc45bc6-ng8ft 3eff5e8d4a69201a430b1ee0cb00cec76dcc59e8002515faae77967b6cae54cc network default NAD default] ADD finished CNI request [tsf-tas/rekor-server-7bcbc45bc6-ng8ft 3eff5e8d4a69201a430b1ee0cb00cec76dcc59e8002515faae77967b6cae54cc network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:e4:96:ef:13:6f\",\"name\":\"3eff5e8d4a69201\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a09361d7-c104-43d2-bd74-d8320224bd6f\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 04:09:31.604391 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0331 04:09:34.156434 3934 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-rmg52-c2s7c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:09:34.157024 3934 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-rmg52-c2s7c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:09:35.041280 3934 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-rmg52-c2s7c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:09:35.041300 3934 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-rmg52-c2s7c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:09:37.563823 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 W0331 04:09:42.078663 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0331 04:09:42.611920 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0331 04:09:59.336753 3934 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:09:59.337323 3934 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:09:59.356528 3934 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:09:59.356558 3934 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:09:59.363677 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0331 04:09:59.363695 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0331 04:10:00.354497 3934 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-7hc4n-pwqkr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:00.355170 3934 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-7hc4n-pwqkr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:01.377875 3934 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-7hc4n-pwqkr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:01.377914 3934 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-7hc4n-pwqkr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:05.778605 3934 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-4nlgf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:05.779258 3934 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-4nlgf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:06.280971 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-4nlgf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:06.280993 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582170-4nlgf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:07.393061 3934 obj_retry.go:464] Detected object tsf/patch-tekton-config-48j5v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:07.393657 3934 obj_retry.go:464] Detected object tsf/patch-tekton-config-48j5v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0331 04:10:08.281507 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0331 04:10:08.415055 3934 obj_retry.go:458] Detected object tsf/patch-tekton-config-48j5v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:08.415078 3934 obj_retry.go:458] Detected object tsf/patch-tekton-config-48j5v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:14.444990 3934 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-kws4f of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:14.445573 3934 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-kws4f of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:14.597576 3934 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0331 04:10:15.349407 3934 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-kws4f of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:15.349433 3934 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-kws4f of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:15.390526 3934 obj_retry.go:458] Detected object tsf/patch-tekton-config-48j5v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:15.390547 3934 obj_retry.go:458] Detected object tsf/patch-tekton-config-48j5v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:15.396499 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-48j5v I0331 04:10:15.396524 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-48j5v I0331 04:10:24.517409 3934 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:24.518031 3934 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:24.535773 3934 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:24.535791 3934 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:24.538995 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0331 04:10:24.539014 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0331 04:10:28.541504 3934 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:28.542067 3934 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:28.583365 3934 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:28.583384 3934 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:28.589397 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0331 04:10:28.589418 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0331 04:10:34.574986 3934 obj_retry.go:464] Detected object tsf-tpa/create-db-95xwk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:34.575567 3934 obj_retry.go:464] Detected object tsf-tpa/create-db-95xwk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:35.597734 3934 obj_retry.go:458] Detected object tsf-tpa/create-db-95xwk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:35.597762 3934 obj_retry.go:458] Detected object tsf-tpa/create-db-95xwk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:38.627469 3934 obj_retry.go:464] Detected object tsf-tpa/migrate-db-fh2bf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:38.628024 3934 obj_retry.go:464] Detected object tsf-tpa/migrate-db-fh2bf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:38.734064 3934 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fh2bf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:38.734087 3934 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fh2bf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:42.549667 3934 obj_retry.go:464] Detected object tsf-tpa/create-importers-6v4x8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:42.550205 3934 obj_retry.go:464] Detected object tsf-tpa/create-importers-6v4x8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:10:43.603340 3934 obj_retry.go:458] Detected object tsf-tpa/create-importers-6v4x8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:43.603365 3934 obj_retry.go:458] Detected object tsf-tpa/create-importers-6v4x8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:10:44.591637 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 W0331 04:10:49.543595 3934 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0331 04:11:17.830448 3934 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:11:17.831040 3934 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:11:17.850305 3934 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:11:17.850319 3934 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:11:17.856136 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0331 04:11:17.856168 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0331 04:11:18.493296 3934 namespace.go:98] [build-service] adding namespace I0331 04:11:18.494275 3934 namespace.go:102] [build-service] adding namespace took 956.629µs I0331 04:11:18.496455 3934 namespace.go:98] [integration-service] adding namespace I0331 04:11:18.497218 3934 namespace.go:102] [integration-service] adding namespace took 749.448µs I0331 04:11:18.516495 3934 namespace.go:98] [image-controller] adding namespace I0331 04:11:18.517220 3934 namespace.go:102] [image-controller] adding namespace took 704.029µs I0331 04:11:18.527694 3934 namespace.go:142] [build-service] updating namespace I0331 04:11:18.558930 3934 namespace.go:142] [build-service] updating namespace I0331 04:11:18.562224 3934 namespace.go:142] [integration-service] updating namespace I0331 04:11:18.603013 3934 namespace.go:142] [image-controller] updating namespace I0331 04:11:18.620574 3934 namespace.go:142] [integration-service] updating namespace I0331 04:11:18.626673 3934 namespace.go:142] [image-controller] updating namespace I0331 04:11:18.684508 3934 namespace.go:142] [build-service] updating namespace I0331 04:11:18.717365 3934 namespace.go:142] [integration-service] updating namespace I0331 04:11:18.758302 3934 namespace.go:98] [release-service] adding namespace I0331 04:11:18.759100 3934 namespace.go:102] [release-service] adding namespace took 780.887µs I0331 04:11:18.793485 3934 namespace.go:142] [release-service] updating namespace I0331 04:11:18.841601 3934 namespace.go:142] [release-service] updating namespace I0331 04:11:18.886335 3934 namespace.go:142] [konflux-ui] updating namespace I0331 04:11:19.051425 3934 namespace.go:98] [konflux-info] adding namespace I0331 04:11:19.052361 3934 namespace.go:102] [konflux-info] adding namespace took 915.288µs I0331 04:11:19.154385 3934 namespace.go:142] [konflux-info] updating namespace I0331 04:11:19.154552 3934 namespace.go:98] [namespace-lister] adding namespace I0331 04:11:19.155498 3934 namespace.go:102] [namespace-lister] adding namespace took 930.734µs I0331 04:11:19.196306 3934 namespace.go:142] [konflux-info] updating namespace I0331 04:11:19.246282 3934 namespace.go:142] [namespace-lister] updating namespace I0331 04:11:19.333016 3934 namespace.go:142] [image-controller] updating namespace I0331 04:11:19.333079 3934 namespace.go:98] [enterprise-contract-service] adding namespace I0331 04:11:19.333174 3934 namespace.go:142] [namespace-lister] updating namespace I0331 04:11:19.333978 3934 namespace.go:102] [enterprise-contract-service] adding namespace took 890.144µs I0331 04:11:19.358466 3934 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:11:19.415513 3934 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:11:19.729343 3934 namespace.go:98] [default-tenant] adding namespace I0331 04:11:19.730179 3934 namespace.go:102] [default-tenant] adding namespace took 814.706µs I0331 04:11:19.838505 3934 namespace.go:98] [segment-bridge] adding namespace I0331 04:11:19.839248 3934 namespace.go:102] [segment-bridge] adding namespace took 725.352µs I0331 04:11:19.993291 3934 namespace.go:142] [default-tenant] updating namespace I0331 04:11:20.207718 3934 namespace.go:142] [default-tenant] updating namespace I0331 04:11:20.271476 3934 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0331 04:11:20.272112 3934 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0331 04:11:20.272135 3934 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0331 04:11:20.274762 3934 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0331 04:11:20.291348 3934 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0331 04:11:20.291371 3934 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0331 04:11:20.292027 3934 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0331 04:11:20.374306 3934 namespace.go:142] [segment-bridge] updating namespace I0331 04:11:20.548730 3934 namespace.go:142] [segment-bridge] updating namespace I0331 04:11:20.827444 3934 namespace.go:142] [build-service] updating namespace I0331 04:11:20.980363 3934 base_network_controller_pods.go:487] [default/konflux-ui/dex-557d86dc9c-kw2rc] creating logical port konflux-ui_dex-557d86dc9c-kw2rc for pod on switch ip-10-0-0-29.ec2.internal I0331 04:11:20.980513 3934 kube.go:256] Updating pod konflux-ui/dex-557d86dc9c-kw2rc I0331 04:11:21.005109 3934 pod.go:62] [konflux-ui/dex-557d86dc9c-kw2rc] pod update took 24.63272ms I0331 04:11:21.005127 3934 base_network_controller_pods.go:937] [default/konflux-ui/dex-557d86dc9c-kw2rc] addLogicalPort annotation time took 24.655242ms I0331 04:11:21.006318 3934 pods.go:271] [konflux-ui/dex-557d86dc9c-kw2rc] addLogicalPort took 25.975832ms, libovsdb time 825.771µs I0331 04:11:21.065801 3934 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w] creating logical port image-controller_image-controller-controller-manager-5f8cd64d55-kvs6w for pod on switch ip-10-0-0-29.ec2.internal I0331 04:11:21.065975 3934 kube.go:256] Updating pod image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w I0331 04:11:21.118490 3934 namespace.go:142] [default-tenant] updating namespace I0331 04:11:21.126696 3934 pod.go:62] [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w] pod update took 60.74835ms I0331 04:11:21.126719 3934 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w] addLogicalPort annotation time took 60.777692ms I0331 04:11:21.127974 3934 pods.go:271] [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w] addLogicalPort took 62.190491ms, libovsdb time 809.22µs I0331 04:11:21.188526 3934 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-759977d69f-nqwwc] creating logical port integration-service_integration-service-controller-manager-759977d69f-nqwwc for pod on switch ip-10-0-0-29.ec2.internal I0331 04:11:21.188632 3934 kube.go:256] Updating pod integration-service/integration-service-controller-manager-759977d69f-nqwwc I0331 04:11:21.259623 3934 pod.go:62] [integration-service/integration-service-controller-manager-759977d69f-nqwwc] pod update took 71.017877ms I0331 04:11:21.259649 3934 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-759977d69f-nqwwc] addLogicalPort annotation time took 71.049697ms I0331 04:11:21.260942 3934 pods.go:271] [integration-service/integration-service-controller-manager-759977d69f-nqwwc] addLogicalPort took 72.437838ms, libovsdb time 864.152µs I0331 04:11:21.458769 3934 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w 1d7e5e2c4bf7df7fc5f131b2e9b17a6684de3ba1cb815b704a50b374a41a5e1e network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w 1d7e5e2c4bf7df7fc5f131b2e9b17a6684de3ba1cb815b704a50b374a41a5e1e network default NAD default] I0331 04:11:21.467558 3934 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-5f8cd64d55-kvs6w, hostIfaceName: 1d7e5e2c4bf7df7, network: default, NAD default, SandboxID: "1d7e5e2c4bf7df7fc5f131b2e9b17a6684de3ba1cb815b704a50b374a41a5e1e", PCI device ID: , UID: "becd9121-1508-4bae-804a-932932a97415", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0331 04:11:21.467577 3934 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7df75dd4df-bndq8] creating logical port build-service_build-service-controller-manager-7df75dd4df-bndq8 for pod on switch ip-10-0-0-29.ec2.internal I0331 04:11:21.467664 3934 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-bndq8 I0331 04:11:21.490413 3934 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:11:21.496026 3934 pod.go:62] [build-service/build-service-controller-manager-7df75dd4df-bndq8] pod update took 28.387318ms I0331 04:11:21.496052 3934 base_network_controller_pods.go:937] [default/build-service/build-service-controller-manager-7df75dd4df-bndq8] addLogicalPort annotation time took 28.419067ms I0331 04:11:21.497442 3934 pods.go:271] [build-service/build-service-controller-manager-7df75dd4df-bndq8] addLogicalPort took 29.87893ms, libovsdb time 950.981µs I0331 04:11:21.519721 3934 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w 1d7e5e2c4bf7df7fc5f131b2e9b17a6684de3ba1cb815b704a50b374a41a5e1e network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w 1d7e5e2c4bf7df7fc5f131b2e9b17a6684de3ba1cb815b704a50b374a41a5e1e network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:9c:40:ad:2b:79\",\"name\":\"1d7e5e2c4bf7df7\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/426063e4-0efd-429c-953c-4ea11de6097f\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:11:21.671557 3934 namespace.go:142] [image-controller] updating namespace I0331 04:11:21.833553 3934 cni.go:403] [build-service/build-service-controller-manager-7df75dd4df-bndq8 d0b40a6356e4f0546c9a06590fbb6b8a37885d98bae4e94fe0038783cb4b34c3 network default NAD default] ADD starting CNI request [build-service/build-service-controller-manager-7df75dd4df-bndq8 d0b40a6356e4f0546c9a06590fbb6b8a37885d98bae4e94fe0038783cb4b34c3 network default NAD default] I0331 04:11:21.842604 3934 helper_linux.go:450] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7df75dd4df-bndq8, hostIfaceName: d0b40a6356e4f05, network: default, NAD default, SandboxID: "d0b40a6356e4f0546c9a06590fbb6b8a37885d98bae4e94fe0038783cb4b34c3", PCI device ID: , UID: "6955bb6e-f6e6-4cec-b3cf-e891ccc75a4a", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0331 04:11:21.877405 3934 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-854f775895-rn77r] creating logical port release-service_release-service-controller-manager-854f775895-rn77r for pod on switch ip-10-0-0-29.ec2.internal I0331 04:11:21.877533 3934 kube.go:256] Updating pod release-service/release-service-controller-manager-854f775895-rn77r I0331 04:11:21.889860 3934 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-f9htd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:11:21.890439 3934 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-f9htd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:11:21.891937 3934 namespace.go:142] [integration-service] updating namespace I0331 04:11:21.892632 3934 cni.go:424] [build-service/build-service-controller-manager-7df75dd4df-bndq8 d0b40a6356e4f0546c9a06590fbb6b8a37885d98bae4e94fe0038783cb4b34c3 network default NAD default] ADD finished CNI request [build-service/build-service-controller-manager-7df75dd4df-bndq8 d0b40a6356e4f0546c9a06590fbb6b8a37885d98bae4e94fe0038783cb4b34c3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:85:f7:4e:50:84\",\"name\":\"d0b40a6356e4f05\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/220c11c1-a37f-4f66-bf17-92cc3396d254\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:11:21.900426 3934 pod.go:62] [release-service/release-service-controller-manager-854f775895-rn77r] pod update took 22.935269ms I0331 04:11:21.900451 3934 base_network_controller_pods.go:937] [default/release-service/release-service-controller-manager-854f775895-rn77r] addLogicalPort annotation time took 22.965234ms I0331 04:11:21.901673 3934 pods.go:271] [release-service/release-service-controller-manager-854f775895-rn77r] addLogicalPort took 24.287242ms, libovsdb time 838.149µs I0331 04:11:21.909498 3934 base_network_controller_pods.go:487] [default/konflux-ui/proxy-64696958d5-cqlpn] creating logical port konflux-ui_proxy-64696958d5-cqlpn for pod on switch ip-10-0-0-29.ec2.internal I0331 04:11:21.909660 3934 kube.go:256] Updating pod konflux-ui/proxy-64696958d5-cqlpn I0331 04:11:21.936944 3934 pod.go:62] [konflux-ui/proxy-64696958d5-cqlpn] pod update took 27.326629ms I0331 04:11:21.936966 3934 base_network_controller_pods.go:937] [default/konflux-ui/proxy-64696958d5-cqlpn] addLogicalPort annotation time took 27.355265ms I0331 04:11:21.938271 3934 pods.go:271] [konflux-ui/proxy-64696958d5-cqlpn] addLogicalPort took 28.787325ms, libovsdb time 898.252µs I0331 04:11:22.094763 3934 namespace.go:142] [konflux-info] updating namespace I0331 04:11:22.266518 3934 namespace.go:142] [namespace-lister] updating namespace I0331 04:11:22.479384 3934 namespace.go:142] [release-service] updating namespace I0331 04:11:22.681556 3934 namespace.go:142] [segment-bridge] updating namespace I0331 04:11:22.798765 3934 namespace.go:142] [build-service] updating namespace I0331 04:11:22.817439 3934 namespace.go:142] [default-tenant] updating namespace I0331 04:11:22.840220 3934 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:11:22.859578 3934 namespace.go:142] [image-controller] updating namespace I0331 04:11:22.875455 3934 namespace.go:142] [integration-service] updating namespace I0331 04:11:22.891248 3934 namespace.go:142] [konflux-info] updating namespace I0331 04:11:22.907323 3934 namespace.go:142] [namespace-lister] updating namespace I0331 04:11:22.926724 3934 namespace.go:142] [release-service] updating namespace I0331 04:11:22.926750 3934 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-f9htd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:11:22.926831 3934 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-f9htd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:11:22.941660 3934 namespace.go:142] [segment-bridge] updating namespace I0331 04:11:22.996581 3934 namespace.go:142] [build-service] updating namespace I0331 04:11:23.048272 3934 namespace.go:142] [default-tenant] updating namespace I0331 04:11:23.124299 3934 namespace.go:142] [enterprise-contract-service] updating namespace I0331 04:11:23.218278 3934 namespace.go:142] [image-controller] updating namespace I0331 04:11:23.308438 3934 namespace.go:142] [integration-service] updating namespace I0331 04:11:23.412210 3934 namespace.go:142] [konflux-info] updating namespace I0331 04:11:23.512514 3934 namespace.go:142] [namespace-lister] updating namespace I0331 04:11:23.609055 3934 namespace.go:142] [release-service] updating namespace I0331 04:11:23.709256 3934 namespace.go:142] [segment-bridge] updating namespace I0331 04:11:24.585591 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 W0331 04:11:25.162788 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0331 04:11:25.184935 3934 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0331 04:11:27.677947 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0331 04:11:29.065655 3934 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-nqwwc 0e3030091963f46d769444ad53ca6e4cbb40ebfc68b8b503168a04386da8ef0b network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-759977d69f-nqwwc 0e3030091963f46d769444ad53ca6e4cbb40ebfc68b8b503168a04386da8ef0b network default NAD default] I0331 04:11:29.074511 3934 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-759977d69f-nqwwc, hostIfaceName: 0e3030091963f46, network: default, NAD default, SandboxID: "0e3030091963f46d769444ad53ca6e4cbb40ebfc68b8b503168a04386da8ef0b", PCI device ID: , UID: "12793e79-2749-4769-9dac-060e0c565c81", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0331 04:11:29.122640 3934 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-nqwwc 0e3030091963f46d769444ad53ca6e4cbb40ebfc68b8b503168a04386da8ef0b network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-759977d69f-nqwwc 0e3030091963f46d769444ad53ca6e4cbb40ebfc68b8b503168a04386da8ef0b network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:07:45:73:7e:d7\",\"name\":\"0e3030091963f46\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/839ae73d-641b-4fcd-be57-7f0298568119\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:11:29.741365 3934 cni.go:403] [release-service/release-service-controller-manager-854f775895-rn77r 5d2f0a98700d54566a77371ed0fb25e097300cc4a88e3efb87806bf95e509c75 network default NAD default] ADD starting CNI request [release-service/release-service-controller-manager-854f775895-rn77r 5d2f0a98700d54566a77371ed0fb25e097300cc4a88e3efb87806bf95e509c75 network default NAD default] I0331 04:11:29.750234 3934 helper_linux.go:450] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-854f775895-rn77r, hostIfaceName: 5d2f0a98700d545, network: default, NAD default, SandboxID: "5d2f0a98700d54566a77371ed0fb25e097300cc4a88e3efb87806bf95e509c75", PCI device ID: , UID: "ddd2450e-c77a-42b4-8300-e8e57894ca42", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0331 04:11:29.781315 3934 cni.go:403] [konflux-ui/proxy-64696958d5-cqlpn b790dbd7f18cb9e3c66ec00e19db0dc6db423abbddcfb2905d77615d893d33d7 network default NAD default] ADD starting CNI request [konflux-ui/proxy-64696958d5-cqlpn b790dbd7f18cb9e3c66ec00e19db0dc6db423abbddcfb2905d77615d893d33d7 network default NAD default] I0331 04:11:29.791184 3934 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-64696958d5-cqlpn, hostIfaceName: b790dbd7f18cb9e, network: default, NAD default, SandboxID: "b790dbd7f18cb9e3c66ec00e19db0dc6db423abbddcfb2905d77615d893d33d7", PCI device ID: , UID: "12ce30dd-40c8-42a5-96b8-cf61333f98b7", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0331 04:11:29.793104 3934 cni.go:403] [konflux-ui/dex-557d86dc9c-kw2rc 912357d35454003e0887828e26fffb65f13b5accca377e3714c454b0da6ad8f2 network default NAD default] ADD starting CNI request [konflux-ui/dex-557d86dc9c-kw2rc 912357d35454003e0887828e26fffb65f13b5accca377e3714c454b0da6ad8f2 network default NAD default] I0331 04:11:29.803516 3934 cni.go:424] [release-service/release-service-controller-manager-854f775895-rn77r 5d2f0a98700d54566a77371ed0fb25e097300cc4a88e3efb87806bf95e509c75 network default NAD default] ADD finished CNI request [release-service/release-service-controller-manager-854f775895-rn77r 5d2f0a98700d54566a77371ed0fb25e097300cc4a88e3efb87806bf95e509c75 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:2e:08:19:4c:65\",\"name\":\"5d2f0a98700d545\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c3919dfa-af3d-40d4-ba53-9b173bc2d071\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:11:29.803602 3934 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: dex-557d86dc9c-kw2rc, hostIfaceName: 912357d35454003, network: default, NAD default, SandboxID: "912357d35454003e0887828e26fffb65f13b5accca377e3714c454b0da6ad8f2", PCI device ID: , UID: "46dc2ea8-37ec-4126-8822-7be54e4e08a9", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0331 04:11:29.830602 3934 namespace.go:142] [openshift-pipelines] updating namespace I0331 04:11:29.844144 3934 cni.go:424] [konflux-ui/proxy-64696958d5-cqlpn b790dbd7f18cb9e3c66ec00e19db0dc6db423abbddcfb2905d77615d893d33d7 network default NAD default] ADD finished CNI request [konflux-ui/proxy-64696958d5-cqlpn b790dbd7f18cb9e3c66ec00e19db0dc6db423abbddcfb2905d77615d893d33d7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:fc:6c:66:58:14\",\"name\":\"b790dbd7f18cb9e\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3484b365-cb85-4826-93c7-2171246fc155\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:11:29.871273 3934 cni.go:424] [konflux-ui/dex-557d86dc9c-kw2rc 912357d35454003e0887828e26fffb65f13b5accca377e3714c454b0da6ad8f2 network default NAD default] ADD finished CNI request [konflux-ui/dex-557d86dc9c-kw2rc 912357d35454003e0887828e26fffb65f13b5accca377e3714c454b0da6ad8f2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:e1:11:20:b7:c5\",\"name\":\"912357d35454003\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cae822a3-029e-4455-8ff2-375d09439816\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 04:11:36.184260 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0331 04:11:36.246871 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0331 04:11:36.247984 3934 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0331 04:11:36.280570 3934 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:11:36.284169 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:11:36.332271 3934 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0331 04:11:36.332287 3934 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0331 04:11:37.247995 3934 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0331 04:11:37.753520 3934 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0331 04:11:37.753535 3934 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0331 04:11:37.772676 3934 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:11:37.774267 3934 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:11:47.256358 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0331 04:11:47.258291 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0331 04:11:47.273787 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0331 04:11:47.276790 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0331 04:11:48.948014 3934 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-nqwwc 0e3030091963f46d769444ad53ca6e4cbb40ebfc68b8b503168a04386da8ef0b network default NAD default] DEL starting CNI request [integration-service/integration-service-controller-manager-759977d69f-nqwwc 0e3030091963f46d769444ad53ca6e4cbb40ebfc68b8b503168a04386da8ef0b network default NAD default] I0331 04:11:49.026559 3934 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-nqwwc 0e3030091963f46d769444ad53ca6e4cbb40ebfc68b8b503168a04386da8ef0b network default NAD default] DEL finished CNI request [integration-service/integration-service-controller-manager-759977d69f-nqwwc 0e3030091963f46d769444ad53ca6e4cbb40ebfc68b8b503168a04386da8ef0b network default NAD default], result "{}", err I0331 04:11:49.112109 3934 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-nqwwc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:11:49.112149 3934 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-759977d69f-nqwwc I0331 04:11:49.113515 3934 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-759977d69f-nqwwc, ips: 10.129.0.38 I0331 04:11:49.113545 3934 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-nqwwc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:11:49.321531 3934 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-nqwwc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:11:49.321557 3934 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-nqwwc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:11:49.324446 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-nqwwc I0331 04:11:49.324472 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-nqwwc W0331 04:11:50.277038 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0331 04:12:13.306975 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0331 04:12:13.306996 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy I0331 04:12:38.603222 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0331 04:12:43.355200 3934 obj_retry.go:464] Detected object konflux-ui/proxy-649fc8d4d8-687sk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:12:43.355789 3934 obj_retry.go:464] Detected object konflux-ui/proxy-649fc8d4d8-687sk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:12:44.142670 3934 obj_retry.go:458] Detected object konflux-ui/proxy-649fc8d4d8-687sk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:12:44.142693 3934 obj_retry.go:458] Detected object konflux-ui/proxy-649fc8d4d8-687sk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:12:44.150775 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-649fc8d4d8-687sk I0331 04:12:44.150804 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-649fc8d4d8-687sk I0331 04:13:09.580876 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0331 04:13:21.573466 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0331 04:13:37.500442 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=403 I0331 04:13:44.490060 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=158 I0331 04:14:10.485685 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0331 04:14:15.487431 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=44 I0331 04:14:17.923839 3934 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-qk5xb for pod on switch ip-10-0-0-29.ec2.internal I0331 04:14:17.923959 3934 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb I0331 04:14:17.963139 3934 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb I0331 04:14:17.977619 3934 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb] pod update took 53.680445ms I0331 04:14:17.977660 3934 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb] addLogicalPort annotation time took 53.723818ms I0331 04:14:17.978929 3934 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb] addLogicalPort took 55.104474ms, libovsdb time 822.479µs I0331 04:14:18.284335 3934 cni.go:403] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb 834cd519069c20aa1ff89c63a36edd136921991ad336db219606e18198d9d1a5 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb 834cd519069c20aa1ff89c63a36edd136921991ad336db219606e18198d9d1a5 network default NAD default] I0331 04:14:18.293118 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-qk5xb, hostIfaceName: 834cd519069c20a, network: default, NAD default, SandboxID: "834cd519069c20aa1ff89c63a36edd136921991ad336db219606e18198d9d1a5", PCI device ID: , UID: "9b9c6de0-758e-409c-80ab-ada8ffe07b7a", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0331 04:14:18.340554 3934 cni.go:424] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb 834cd519069c20aa1ff89c63a36edd136921991ad336db219606e18198d9d1a5 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-qk5xb 834cd519069c20aa1ff89c63a36edd136921991ad336db219606e18198d9d1a5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:93:16:a4:85:d6\",\"name\":\"834cd519069c20a\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4be0818f-9d4f-4374-90e0-6ee86e559488\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0331 04:14:20.719393 3934 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:14:20.719412 3934 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0331 04:14:20.750607 3934 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0331 04:14:23.828274 3934 util.go:959] Failed to build node endpoints for node ip-10-0-0-29.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0331 04:14:35.566128 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0331 04:14:49.040204 3934 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:14:49.040764 3934 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:14:49.079436 3934 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:49.079454 3934 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:14:49.085298 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0331 04:14:49.085326 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0331 04:15:13.603220 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0331 04:15:18.528716 3934 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk f6cdae1592f6baf524a45147e442a44bfeb9cd8bb671d567575a463d1eedf961 network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk f6cdae1592f6baf524a45147e442a44bfeb9cd8bb671d567575a463d1eedf961 network default NAD default] I0331 04:15:18.620590 3934 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk f6cdae1592f6baf524a45147e442a44bfeb9cd8bb671d567575a463d1eedf961 network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk f6cdae1592f6baf524a45147e442a44bfeb9cd8bb671d567575a463d1eedf961 network default NAD default], result "{}", err I0331 04:15:18.681396 3934 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:15:18.681427 3934 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk I0331 04:15:18.682823 3934 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk, ips: 10.129.0.24 I0331 04:15:18.682853 3934 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:15:18.711555 3934 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-56b684c87-7kbsw] creating logical port image-controller_image-controller-controller-manager-56b684c87-7kbsw for pod on switch ip-10-0-0-29.ec2.internal I0331 04:15:18.711646 3934 kube.go:256] Updating pod image-controller/image-controller-controller-manager-56b684c87-7kbsw I0331 04:15:18.726037 3934 pod.go:62] [image-controller/image-controller-controller-manager-56b684c87-7kbsw] pod update took 14.411659ms I0331 04:15:18.726060 3934 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-56b684c87-7kbsw] addLogicalPort annotation time took 14.435932ms I0331 04:15:18.727354 3934 pods.go:271] [image-controller/image-controller-controller-manager-56b684c87-7kbsw] addLogicalPort took 15.810213ms, libovsdb time 840.861µs I0331 04:15:19.072533 3934 cni.go:403] [image-controller/image-controller-controller-manager-56b684c87-7kbsw c6af4bfe57d9b9341c88016fd9da77f23fecbc4bded8ab4d4761f46fc6ed3575 network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-56b684c87-7kbsw c6af4bfe57d9b9341c88016fd9da77f23fecbc4bded8ab4d4761f46fc6ed3575 network default NAD default] I0331 04:15:19.081476 3934 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-56b684c87-7kbsw, hostIfaceName: c6af4bfe57d9b93, network: default, NAD default, SandboxID: "c6af4bfe57d9b9341c88016fd9da77f23fecbc4bded8ab4d4761f46fc6ed3575", PCI device ID: , UID: "87cd3ade-cf1c-497c-b807-99406fc5c6ab", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0331 04:15:19.128875 3934 cni.go:424] [image-controller/image-controller-controller-manager-56b684c87-7kbsw c6af4bfe57d9b9341c88016fd9da77f23fecbc4bded8ab4d4761f46fc6ed3575 network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-56b684c87-7kbsw c6af4bfe57d9b9341c88016fd9da77f23fecbc4bded8ab4d4761f46fc6ed3575 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:80:00:2f:6b:7b\",\"name\":\"c6af4bfe57d9b93\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/562729c3-757b-4b0d-b6b5-29f3d4b4f0e7\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:15:19.451165 3934 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:19.451193 3934 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:19.458208 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk I0331 04:15:19.458235 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-dstmk I0331 04:15:31.649836 3934 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w 1d7e5e2c4bf7df7fc5f131b2e9b17a6684de3ba1cb815b704a50b374a41a5e1e network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w 1d7e5e2c4bf7df7fc5f131b2e9b17a6684de3ba1cb815b704a50b374a41a5e1e network default NAD default] I0331 04:15:31.739681 3934 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w 1d7e5e2c4bf7df7fc5f131b2e9b17a6684de3ba1cb815b704a50b374a41a5e1e network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w 1d7e5e2c4bf7df7fc5f131b2e9b17a6684de3ba1cb815b704a50b374a41a5e1e network default NAD default], result "{}", err I0331 04:15:31.794304 3934 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:15:31.794351 3934 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w I0331 04:15:31.795731 3934 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w, ips: 10.129.0.37 I0331 04:15:31.795763 3934 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:15:32.523257 3934 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:32.523290 3934 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:15:32.530874 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w I0331 04:15:32.530927 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-kvs6w I0331 04:16:24.595311 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0331 04:16:31.566848 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0331 04:16:54.569054 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0331 04:16:56.488469 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=54 I0331 04:16:58.589213 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0331 04:17:05.488297 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=125 I0331 04:18:02.606571 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0331 04:18:11.600474 3934 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 I0331 04:19:14.577084 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0331 04:20:00.212344 3934 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29582180-h62f5 for pod on switch ip-10-0-0-29.ec2.internal I0331 04:20:00.212458 3934 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 I0331 04:20:00.238066 3934 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5] pod update took 25.630788ms I0331 04:20:00.238090 3934 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5] addLogicalPort annotation time took 25.659719ms I0331 04:20:00.239427 3934 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5] addLogicalPort took 27.098393ms, libovsdb time 895.355µs I0331 04:20:00.574087 3934 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 38d0ac25b10ec2c2804be23f88376695dc2becb3445ee86d658ef905d5ba23c8 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 38d0ac25b10ec2c2804be23f88376695dc2becb3445ee86d658ef905d5ba23c8 network default NAD default] I0331 04:20:00.582564 3934 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29582180-h62f5, hostIfaceName: 38d0ac25b10ec2c, network: default, NAD default, SandboxID: "38d0ac25b10ec2c2804be23f88376695dc2becb3445ee86d658ef905d5ba23c8", PCI device ID: , UID: "a157f5de-1c1a-4ac6-9758-199949306644", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0331 04:20:00.838536 3934 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 38d0ac25b10ec2c2804be23f88376695dc2becb3445ee86d658ef905d5ba23c8 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 38d0ac25b10ec2c2804be23f88376695dc2becb3445ee86d658ef905d5ba23c8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"12:b0:f0:99:69:a4\",\"name\":\"38d0ac25b10ec2c\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/912b7e87-e804-418b-9706-311e94c3e4e1\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:20:03.941514 3934 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 38d0ac25b10ec2c2804be23f88376695dc2becb3445ee86d658ef905d5ba23c8 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 38d0ac25b10ec2c2804be23f88376695dc2becb3445ee86d658ef905d5ba23c8 network default NAD default] I0331 04:20:04.027022 3934 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 38d0ac25b10ec2c2804be23f88376695dc2becb3445ee86d658ef905d5ba23c8 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 38d0ac25b10ec2c2804be23f88376695dc2becb3445ee86d658ef905d5ba23c8 network default NAD default], result "{}", err I0331 04:20:04.078193 3934 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:20:04.078222 3934 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 I0331 04:20:04.079857 3934 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5, ips: 10.129.0.44 I0331 04:20:04.079912 3934 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:20:04.950546 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:04.950592 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582180-h62f5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:23.584659 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0331 04:20:35.045483 3934 obj_retry.go:458] Detected object tsf-tpa/create-db-95xwk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:35.045532 3934 obj_retry.go:458] Detected object tsf-tpa/create-db-95xwk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:35.050445 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-95xwk I0331 04:20:35.050470 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-95xwk I0331 04:20:38.032491 3934 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fh2bf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:38.032537 3934 obj_retry.go:458] Detected object tsf-tpa/migrate-db-fh2bf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:38.037950 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-fh2bf I0331 04:20:38.037976 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-fh2bf I0331 04:20:40.504630 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=70 I0331 04:20:43.029231 3934 obj_retry.go:458] Detected object tsf-tpa/create-importers-6v4x8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:43.029275 3934 obj_retry.go:458] Detected object tsf-tpa/create-importers-6v4x8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:20:43.033827 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-6v4x8 I0331 04:20:43.033852 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-6v4x8 I0331 04:21:01.490737 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=35 I0331 04:21:09.572435 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0331 04:21:25.607297 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0331 04:22:39.489358 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0331 04:23:02.570881 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0331 04:23:09.494127 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=11 I0331 04:24:15.592915 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0331 04:24:36.598641 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0331 04:24:52.573359 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0331 04:25:01.610513 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0331 04:25:14.492448 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=10 I0331 04:25:30.492506 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=9 I0331 04:26:49.508600 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=9 I0331 04:26:52.146010 3934 namespace.go:98] [default-tenant-managed] adding namespace I0331 04:26:52.146941 3934 namespace.go:102] [default-tenant-managed] adding namespace took 908.978µs I0331 04:26:52.166060 3934 namespace.go:142] [default-tenant-managed] updating namespace I0331 04:26:52.181546 3934 namespace.go:142] [default-tenant-managed] updating namespace I0331 04:26:53.156872 3934 namespace.go:142] [default-tenant-managed] updating namespace I0331 04:26:53.301618 3934 namespace.go:142] [default-tenant-managed] updating namespace I0331 04:26:53.388625 3934 namespace.go:142] [default-tenant-managed] updating namespace I0331 04:27:13.581205 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0331 04:27:38.851935 3934 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod] creating logical port default-tenant_tsf-comp-hwcq-on-push-5drxx-init-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:27:38.852103 3934 kube.go:256] Updating pod default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod I0331 04:27:38.880735 3934 pod.go:62] [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod] pod update took 28.680472ms I0331 04:27:38.880758 3934 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod] addLogicalPort annotation time took 28.710042ms I0331 04:27:38.882126 3934 pods.go:271] [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod] addLogicalPort took 30.233061ms, libovsdb time 936.924µs I0331 04:27:39.215576 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod e6fbf01d039f6fc029d36dab742c284862812296058433639373aeb83ca642cb network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod e6fbf01d039f6fc029d36dab742c284862812296058433639373aeb83ca642cb network default NAD default] I0331 04:27:39.224637 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-hwcq-on-push-5drxx-init-pod, hostIfaceName: e6fbf01d039f6fc, network: default, NAD default, SandboxID: "e6fbf01d039f6fc029d36dab742c284862812296058433639373aeb83ca642cb", PCI device ID: , UID: "f71aa8a3-6129-469a-9982-8075d529e517", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0331 04:27:39.480388 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod e6fbf01d039f6fc029d36dab742c284862812296058433639373aeb83ca642cb network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod e6fbf01d039f6fc029d36dab742c284862812296058433639373aeb83ca642cb network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:a5:29:7c:e8:b6\",\"name\":\"e6fbf01d039f6fc\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b3683781-a9df-4779-a3f3-6d2194a1c360\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:27:55.450448 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod e6fbf01d039f6fc029d36dab742c284862812296058433639373aeb83ca642cb network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod e6fbf01d039f6fc029d36dab742c284862812296058433639373aeb83ca642cb network default NAD default] I0331 04:27:55.535024 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod e6fbf01d039f6fc029d36dab742c284862812296058433639373aeb83ca642cb network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod e6fbf01d039f6fc029d36dab742c284862812296058433639373aeb83ca642cb network default NAD default], result "{}", err I0331 04:27:55.659318 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:27:55.659355 3934 pods.go:176] Deleting pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod I0331 04:27:55.660683 3934 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod, ips: 10.129.0.45 I0331 04:27:55.660711 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-init-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:27:56.273331 3934 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod] creating logical port default-tenant_tsf-comp-hwcq-on-push-5drxx-clone-repository-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:27:56.273512 3934 kube.go:256] Updating pod default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod I0331 04:27:56.300707 3934 pod.go:62] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod] pod update took 27.231712ms I0331 04:27:56.300727 3934 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod] addLogicalPort annotation time took 27.255998ms I0331 04:27:56.302225 3934 pods.go:271] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod] addLogicalPort took 28.90852ms, libovsdb time 1.089269ms I0331 04:27:56.696001 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod cc11278d3cd4a58dd6b2d3c2252095c31610379587bd9a7b232e0b81878c43ce network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod cc11278d3cd4a58dd6b2d3c2252095c31610379587bd9a7b232e0b81878c43ce network default NAD default] I0331 04:27:56.704840 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-hwcq-on-push-5drxx-clone-repository-pod, hostIfaceName: cc11278d3cd4a58, network: default, NAD default, SandboxID: "cc11278d3cd4a58dd6b2d3c2252095c31610379587bd9a7b232e0b81878c43ce", PCI device ID: , UID: "61ab5d20-71d4-4ea9-9e63-804ce79cd28d", MAC: 0a:58:0a:81:00:2e, IPs: [10.129.0.46/23] I0331 04:27:56.755357 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod cc11278d3cd4a58dd6b2d3c2252095c31610379587bd9a7b232e0b81878c43ce network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod cc11278d3cd4a58dd6b2d3c2252095c31610379587bd9a7b232e0b81878c43ce network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:e1:1e:15:99:66\",\"name\":\"cc11278d3cd4a58\"},{\"mac\":\"0a:58:0a:81:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e7fd6e76-3625-45e8-b785-e3ff8077350d\"}],\"ips\":[{\"address\":\"10.129.0.46/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:27:58.576599 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0331 04:28:09.604000 3934 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0331 04:28:22.627676 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod cc11278d3cd4a58dd6b2d3c2252095c31610379587bd9a7b232e0b81878c43ce network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod cc11278d3cd4a58dd6b2d3c2252095c31610379587bd9a7b232e0b81878c43ce network default NAD default] I0331 04:28:22.721359 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod cc11278d3cd4a58dd6b2d3c2252095c31610379587bd9a7b232e0b81878c43ce network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod cc11278d3cd4a58dd6b2d3c2252095c31610379587bd9a7b232e0b81878c43ce network default NAD default], result "{}", err I0331 04:28:22.906756 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:28:22.906807 3934 pods.go:176] Deleting pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod I0331 04:28:22.908196 3934 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod, ips: 10.129.0.46 I0331 04:28:22.908229 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-clone-repository-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:28:27.492995 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0331 04:28:43.944256 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-prefetch-dependencies-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:28:43.944823 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-prefetch-dependencies-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:28:44.251503 3934 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod] creating logical port default-tenant_tsf-comp-hwcq-on-push-5drxx-build-container-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:28:44.251675 3934 kube.go:256] Updating pod default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod I0331 04:28:44.301206 3934 pod.go:62] [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod] pod update took 49.5691ms I0331 04:28:44.301234 3934 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod] addLogicalPort annotation time took 49.601023ms I0331 04:28:44.302862 3934 pods.go:271] [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod] addLogicalPort took 51.374216ms, libovsdb time 1.087971ms I0331 04:28:44.625471 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod fa14f622bb7bbc2c3b179729fdcd4000959c520c18e64e29ed6c751d7b588ab4 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod fa14f622bb7bbc2c3b179729fdcd4000959c520c18e64e29ed6c751d7b588ab4 network default NAD default] I0331 04:28:44.635062 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-hwcq-on-push-5drxx-build-container-pod, hostIfaceName: fa14f622bb7bbc2, network: default, NAD default, SandboxID: "fa14f622bb7bbc2c3b179729fdcd4000959c520c18e64e29ed6c751d7b588ab4", PCI device ID: , UID: "aad0a6cb-bb66-4b5f-9a63-6e3a2fe7635e", MAC: 0a:58:0a:81:00:2f, IPs: [10.129.0.47/23] I0331 04:28:44.685081 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod fa14f622bb7bbc2c3b179729fdcd4000959c520c18e64e29ed6c751d7b588ab4 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod fa14f622bb7bbc2c3b179729fdcd4000959c520c18e64e29ed6c751d7b588ab4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:7c:8e:1f:a5:93\",\"name\":\"fa14f622bb7bbc2\"},{\"mac\":\"0a:58:0a:81:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba4d9efd-d2d5-48aa-8baa-3fc9b97767cc\"}],\"ips\":[{\"address\":\"10.129.0.47/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:29:20.497449 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0331 04:29:21.611039 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0331 04:30:01.588299 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0331 04:30:04.590213 3934 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582190-sllnw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:30:04.590794 3934 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582190-sllnw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:30:05.483274 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582190-sllnw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:30:05.483300 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582190-sllnw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:30:05.513044 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-vt4j6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:30:05.513085 3934 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-vt4j6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:30:05.518724 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-vt4j6 I0331 04:30:05.518744 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-backplane/osd-delete-backplane-serviceaccounts-29582160-vt4j6 I0331 04:30:35.577658 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 W0331 04:30:38.882710 3934 duration.go:461] Config duration recorder: measurement expired for pod/tsf-tpa/migrate-db-fh2bf I0331 04:30:55.494582 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=43 I0331 04:30:56.599268 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod fa14f622bb7bbc2c3b179729fdcd4000959c520c18e64e29ed6c751d7b588ab4 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod fa14f622bb7bbc2c3b179729fdcd4000959c520c18e64e29ed6c751d7b588ab4 network default NAD default] I0331 04:30:56.697650 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod fa14f622bb7bbc2c3b179729fdcd4000959c520c18e64e29ed6c751d7b588ab4 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod fa14f622bb7bbc2c3b179729fdcd4000959c520c18e64e29ed6c751d7b588ab4 network default NAD default], result "{}", err I0331 04:30:56.818769 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:30:56.818810 3934 pods.go:176] Deleting pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod I0331 04:30:56.820127 3934 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod, ips: 10.129.0.47 I0331 04:30:56.820152 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-build-container-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:30:59.596380 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0331 04:31:15.602059 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0331 04:31:36.513354 3934 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod] creating logical port default-tenant_tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:31:36.513466 3934 kube.go:256] Updating pod default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod I0331 04:31:36.546068 3934 pod.go:62] [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod] pod update took 32.629929ms I0331 04:31:36.546092 3934 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod] addLogicalPort annotation time took 32.659988ms I0331 04:31:36.547375 3934 pods.go:271] [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod] addLogicalPort took 34.034613ms, libovsdb time 866.97µs I0331 04:31:36.572679 3934 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod] creating logical port default-tenant_tsf-comp-hwcq-on-push-5drxx-clair-scan-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:31:36.572848 3934 kube.go:256] Updating pod default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod I0331 04:31:36.608966 3934 pod.go:62] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod] pod update took 36.154685ms I0331 04:31:36.608987 3934 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod] addLogicalPort annotation time took 36.180604ms I0331 04:31:36.610197 3934 pods.go:271] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod] addLogicalPort took 37.535821ms, libovsdb time 831.24µs I0331 04:31:36.878069 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod e9308c65393a32e52e67bee8a9ec03f149b7513c9810e1a4104fd79da0a6ec76 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod e9308c65393a32e52e67bee8a9ec03f149b7513c9810e1a4104fd79da0a6ec76 network default NAD default] I0331 04:31:36.886819 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod, hostIfaceName: e9308c65393a32e, network: default, NAD default, SandboxID: "e9308c65393a32e52e67bee8a9ec03f149b7513c9810e1a4104fd79da0a6ec76", PCI device ID: , UID: "908c7a8b-4a1e-4a22-b965-603634305a21", MAC: 0a:58:0a:81:00:30, IPs: [10.129.0.48/23] I0331 04:31:36.940523 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod d975c299a6af08a2e1a1a6eb2d3d7817a155278b6e3c1ca695a9e57276d8b24a network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod d975c299a6af08a2e1a1a6eb2d3d7817a155278b6e3c1ca695a9e57276d8b24a network default NAD default] I0331 04:31:36.950341 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-hwcq-on-push-5drxx-clair-scan-pod, hostIfaceName: d975c299a6af08a, network: default, NAD default, SandboxID: "d975c299a6af08a2e1a1a6eb2d3d7817a155278b6e3c1ca695a9e57276d8b24a", PCI device ID: , UID: "3e1b805f-b816-48aa-bf3c-d7e3bc58d72b", MAC: 0a:58:0a:81:00:31, IPs: [10.129.0.49/23] I0331 04:31:36.998247 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod d975c299a6af08a2e1a1a6eb2d3d7817a155278b6e3c1ca695a9e57276d8b24a network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod d975c299a6af08a2e1a1a6eb2d3d7817a155278b6e3c1ca695a9e57276d8b24a network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:98:db:c7:d8:b5\",\"name\":\"d975c299a6af08a\"},{\"mac\":\"0a:58:0a:81:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cd0f768c-221c-4f69-88c7-78e0df5c7de6\"}],\"ips\":[{\"address\":\"10.129.0.49/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:31:37.067082 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-build-image-index-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:31:37.067645 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-build-image-index-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:31:37.143801 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod e9308c65393a32e52e67bee8a9ec03f149b7513c9810e1a4104fd79da0a6ec76 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod e9308c65393a32e52e67bee8a9ec03f149b7513c9810e1a4104fd79da0a6ec76 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:fd:44:10:fd:0e\",\"name\":\"e9308c65393a32e\"},{\"mac\":\"0a:58:0a:81:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/780c433c-4526-445a-94f5-ccdc0bb49a27\"}],\"ips\":[{\"address\":\"10.129.0.48/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:31:37.309618 3934 base_network_controller_pods.go:487] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod] creating logical port default-tenant_tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:31:37.309767 3934 kube.go:256] Updating pod default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod I0331 04:31:37.334795 3934 pod.go:62] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod] pod update took 25.059966ms I0331 04:31:37.334813 3934 base_network_controller_pods.go:937] [default/default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod] addLogicalPort annotation time took 25.083663ms I0331 04:31:37.336160 3934 pods.go:271] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod] addLogicalPort took 26.561901ms, libovsdb time 908.802µs I0331 04:31:37.705149 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod f982b66def32c84df9e72b266c493cb4b21fba32bc31fcf007d910d4a0e91843 network default NAD default] ADD starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod f982b66def32c84df9e72b266c493cb4b21fba32bc31fcf007d910d4a0e91843 network default NAD default] I0331 04:31:37.714368 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod, hostIfaceName: f982b66def32c84, network: default, NAD default, SandboxID: "f982b66def32c84df9e72b266c493cb4b21fba32bc31fcf007d910d4a0e91843", PCI device ID: , UID: "28c9968f-23ac-46f6-b48c-00e43891a4b5", MAC: 0a:58:0a:81:00:32, IPs: [10.129.0.50/23] I0331 04:31:37.762838 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod f982b66def32c84df9e72b266c493cb4b21fba32bc31fcf007d910d4a0e91843 network default NAD default] ADD finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod f982b66def32c84df9e72b266c493cb4b21fba32bc31fcf007d910d4a0e91843 network default NAD default], result "{\"interfaces\":[{\"mac\":\"32:5e:64:26:ce:dd\",\"name\":\"f982b66def32c84\"},{\"mac\":\"0a:58:0a:81:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a27a72bc-5750-44fa-b630-a9cf1ac50580\"}],\"ips\":[{\"address\":\"10.129.0.50/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:31:58.614330 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0331 04:32:35.585363 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-coverity-availability-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:35.585995 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-coverity-availability-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:35.629330 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-apply-tags-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:35.629958 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-apply-tags-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:36.426486 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-sast-unicode-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:36.427101 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-sast-unicode-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:37.787524 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-sast-snyk-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:37.788192 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-sast-snyk-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:39.336910 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod e9308c65393a32e52e67bee8a9ec03f149b7513c9810e1a4104fd79da0a6ec76 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod e9308c65393a32e52e67bee8a9ec03f149b7513c9810e1a4104fd79da0a6ec76 network default NAD default] I0331 04:32:39.445917 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod e9308c65393a32e52e67bee8a9ec03f149b7513c9810e1a4104fd79da0a6ec76 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod e9308c65393a32e52e67bee8a9ec03f149b7513c9810e1a4104fd79da0a6ec76 network default NAD default], result "{}", err I0331 04:32:39.539037 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:39.539074 3934 pods.go:176] Deleting pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod I0331 04:32:39.540544 3934 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod, ips: 10.129.0.48 I0331 04:32:39.540575 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-deprecated-base-image-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:48.403697 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod f982b66def32c84df9e72b266c493cb4b21fba32bc31fcf007d910d4a0e91843 network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod f982b66def32c84df9e72b266c493cb4b21fba32bc31fcf007d910d4a0e91843 network default NAD default] I0331 04:32:48.496361 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod f982b66def32c84df9e72b266c493cb4b21fba32bc31fcf007d910d4a0e91843 network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod f982b66def32c84df9e72b266c493cb4b21fba32bc31fcf007d910d4a0e91843 network default NAD default], result "{}", err I0331 04:32:48.646358 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:48.646400 3934 pods.go:176] Deleting pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod I0331 04:32:48.648390 3934 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod, ips: 10.129.0.50 I0331 04:32:48.648424 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-clamav-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:49.885684 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-sast-shell-check-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:49.886344 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-sast-shell-check-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:49.913338 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-push-dockerfile-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:49.913860 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-push-dockerfile-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:32:54.574213 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0331 04:33:13.512278 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=215 I0331 04:33:18.096195 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-ecosystem-cert-preflight-checks-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:33:18.096695 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-ecosystem-cert-preflight-checks-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:33:35.236056 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-rpms-signature-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:33:35.236674 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-rpms-signature-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:33:42.606991 3934 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=5 I0331 04:33:43.820709 3934 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-xps7l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:33:43.821302 3934 obj_retry.go:464] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-xps7l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:33:44.093532 3934 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-xps7l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:33:44.093559 3934 obj_retry.go:458] Detected object openshift-pipelines/tekton-pipelines-webhook-584c448755-xps7l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:33:44.100131 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-pipelines/tekton-pipelines-webhook-584c448755-xps7l I0331 04:33:44.100171 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-pipelines/tekton-pipelines-webhook-584c448755-xps7l I0331 04:34:19.495574 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=15 I0331 04:34:20.497023 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=6 I0331 04:34:42.496213 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=11 I0331 04:35:02.277862 3934 cni.go:403] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod d975c299a6af08a2e1a1a6eb2d3d7817a155278b6e3c1ca695a9e57276d8b24a network default NAD default] DEL starting CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod d975c299a6af08a2e1a1a6eb2d3d7817a155278b6e3c1ca695a9e57276d8b24a network default NAD default] I0331 04:35:02.375548 3934 cni.go:424] [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod d975c299a6af08a2e1a1a6eb2d3d7817a155278b6e3c1ca695a9e57276d8b24a network default NAD default] DEL finished CNI request [default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod d975c299a6af08a2e1a1a6eb2d3d7817a155278b6e3c1ca695a9e57276d8b24a network default NAD default], result "{}", err I0331 04:35:02.456420 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:35:02.456450 3934 pods.go:176] Deleting pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod I0331 04:35:02.457711 3934 pods.go:236] Attempting to release IPs for pod: default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod, ips: 10.129.0.49 I0331 04:35:02.457736 3934 obj_retry.go:464] Detected object default-tenant/tsf-comp-hwcq-on-push-5drxx-clair-scan-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:35:05.594881 3934 base_network_controller_pods.go:487] [default/default-tenant/my-integration-test-frqx-652nn-verify-pod] creating logical port default-tenant_my-integration-test-frqx-652nn-verify-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:35:05.595124 3934 kube.go:256] Updating pod default-tenant/my-integration-test-frqx-652nn-verify-pod I0331 04:35:05.644120 3934 pod.go:62] [default-tenant/my-integration-test-frqx-652nn-verify-pod] pod update took 49.05591ms I0331 04:35:05.644142 3934 base_network_controller_pods.go:937] [default/default-tenant/my-integration-test-frqx-652nn-verify-pod] addLogicalPort annotation time took 49.082559ms I0331 04:35:05.645486 3934 pods.go:271] [default-tenant/my-integration-test-frqx-652nn-verify-pod] addLogicalPort took 50.618143ms, libovsdb time 909.539µs I0331 04:35:05.965528 3934 cni.go:403] [default-tenant/my-integration-test-frqx-652nn-verify-pod a30706e63115b16d11da7c59989c73b26544e34f1687dbf96bc0b75748e222d6 network default NAD default] ADD starting CNI request [default-tenant/my-integration-test-frqx-652nn-verify-pod a30706e63115b16d11da7c59989c73b26544e34f1687dbf96bc0b75748e222d6 network default NAD default] I0331 04:35:05.974947 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant, podName: my-integration-test-frqx-652nn-verify-pod, hostIfaceName: a30706e63115b16, network: default, NAD default, SandboxID: "a30706e63115b16d11da7c59989c73b26544e34f1687dbf96bc0b75748e222d6", PCI device ID: , UID: "3bfebdec-c31d-4704-b0a5-c9e76adeb90e", MAC: 0a:58:0a:81:00:33, IPs: [10.129.0.51/23] I0331 04:35:06.024362 3934 cni.go:424] [default-tenant/my-integration-test-frqx-652nn-verify-pod a30706e63115b16d11da7c59989c73b26544e34f1687dbf96bc0b75748e222d6 network default NAD default] ADD finished CNI request [default-tenant/my-integration-test-frqx-652nn-verify-pod a30706e63115b16d11da7c59989c73b26544e34f1687dbf96bc0b75748e222d6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:a1:20:42:8b:ac\",\"name\":\"a30706e63115b16\"},{\"mac\":\"0a:58:0a:81:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4317f596-c3fd-4c60-a9fb-bf023740ce93\"}],\"ips\":[{\"address\":\"10.129.0.51/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:35:17.591333 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0331 04:35:23.580310 3934 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0331 04:35:27.480527 3934 cni.go:403] [default-tenant/my-integration-test-frqx-652nn-verify-pod a30706e63115b16d11da7c59989c73b26544e34f1687dbf96bc0b75748e222d6 network default NAD default] DEL starting CNI request [default-tenant/my-integration-test-frqx-652nn-verify-pod a30706e63115b16d11da7c59989c73b26544e34f1687dbf96bc0b75748e222d6 network default NAD default] I0331 04:35:27.600746 3934 cni.go:424] [default-tenant/my-integration-test-frqx-652nn-verify-pod a30706e63115b16d11da7c59989c73b26544e34f1687dbf96bc0b75748e222d6 network default NAD default] DEL finished CNI request [default-tenant/my-integration-test-frqx-652nn-verify-pod a30706e63115b16d11da7c59989c73b26544e34f1687dbf96bc0b75748e222d6 network default NAD default], result "{}", err I0331 04:35:27.715146 3934 obj_retry.go:464] Detected object default-tenant/my-integration-test-frqx-652nn-verify-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:35:27.715209 3934 pods.go:176] Deleting pod: default-tenant/my-integration-test-frqx-652nn-verify-pod I0331 04:35:27.716673 3934 pods.go:236] Attempting to release IPs for pod: default-tenant/my-integration-test-frqx-652nn-verify-pod, ips: 10.129.0.51 I0331 04:35:27.716702 3934 obj_retry.go:464] Detected object default-tenant/my-integration-test-frqx-652nn-verify-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:35:41.034484 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod] creating logical port default-tenant-managed_managed-f7ftt-verify-access-to-resources-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:35:41.034604 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod I0331 04:35:41.054516 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod] pod update took 19.943751ms I0331 04:35:41.054541 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod] addLogicalPort annotation time took 19.970835ms I0331 04:35:41.056181 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod] addLogicalPort took 21.719488ms, libovsdb time 1.190501ms I0331 04:35:41.405639 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod 577fde9a3d5dd04bc97a1ab5909d3bc7cab38eef37d64292db74f72c418b6e74 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod 577fde9a3d5dd04bc97a1ab5909d3bc7cab38eef37d64292db74f72c418b6e74 network default NAD default] I0331 04:35:41.414334 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-verify-access-to-resources-pod, hostIfaceName: 577fde9a3d5dd04, network: default, NAD default, SandboxID: "577fde9a3d5dd04bc97a1ab5909d3bc7cab38eef37d64292db74f72c418b6e74", PCI device ID: , UID: "502f56c0-67d8-4a20-8335-754b6ec00260", MAC: 0a:58:0a:81:00:34, IPs: [10.129.0.52/23] I0331 04:35:41.670975 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod 577fde9a3d5dd04bc97a1ab5909d3bc7cab38eef37d64292db74f72c418b6e74 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod 577fde9a3d5dd04bc97a1ab5909d3bc7cab38eef37d64292db74f72c418b6e74 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:b2:9c:10:80:88\",\"name\":\"577fde9a3d5dd04\"},{\"mac\":\"0a:58:0a:81:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/32920759-eaa8-49f4-9b19-cdab4757e33e\"}],\"ips\":[{\"address\":\"10.129.0.52/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:35:43.832460 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/affinity-assistant-b07ee9c152-0] creating logical port default-tenant-managed_affinity-assistant-b07ee9c152-0 for pod on switch ip-10-0-0-29.ec2.internal I0331 04:35:43.832589 3934 kube.go:256] Updating pod default-tenant-managed/affinity-assistant-b07ee9c152-0 I0331 04:35:43.854365 3934 pod.go:62] [default-tenant-managed/affinity-assistant-b07ee9c152-0] pod update took 21.800774ms I0331 04:35:43.854394 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/affinity-assistant-b07ee9c152-0] addLogicalPort annotation time took 21.833728ms I0331 04:35:43.855780 3934 pods.go:271] [default-tenant-managed/affinity-assistant-b07ee9c152-0] addLogicalPort took 23.342079ms, libovsdb time 988.611µs I0331 04:35:44.204843 3934 cni.go:403] [default-tenant-managed/affinity-assistant-b07ee9c152-0 a4841ce4b88194a1891ae32c62137b87403bb25492cd0b56e1d6879ee0e588a0 network default NAD default] ADD starting CNI request [default-tenant-managed/affinity-assistant-b07ee9c152-0 a4841ce4b88194a1891ae32c62137b87403bb25492cd0b56e1d6879ee0e588a0 network default NAD default] I0331 04:35:44.213773 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: affinity-assistant-b07ee9c152-0, hostIfaceName: a4841ce4b88194a, network: default, NAD default, SandboxID: "a4841ce4b88194a1891ae32c62137b87403bb25492cd0b56e1d6879ee0e588a0", PCI device ID: , UID: "d1d2388a-40ae-4e1c-b5b3-9f42d449b633", MAC: 0a:58:0a:81:00:35, IPs: [10.129.0.53/23] I0331 04:35:44.272189 3934 cni.go:424] [default-tenant-managed/affinity-assistant-b07ee9c152-0 a4841ce4b88194a1891ae32c62137b87403bb25492cd0b56e1d6879ee0e588a0 network default NAD default] ADD finished CNI request [default-tenant-managed/affinity-assistant-b07ee9c152-0 a4841ce4b88194a1891ae32c62137b87403bb25492cd0b56e1d6879ee0e588a0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:f0:62:55:28:2a\",\"name\":\"a4841ce4b88194a\"},{\"mac\":\"0a:58:0a:81:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2dff2fb-9d85-4072-9560-2833de2aea87\"}],\"ips\":[{\"address\":\"10.129.0.53/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:35:59.726626 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod 577fde9a3d5dd04bc97a1ab5909d3bc7cab38eef37d64292db74f72c418b6e74 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod 577fde9a3d5dd04bc97a1ab5909d3bc7cab38eef37d64292db74f72c418b6e74 network default NAD default] I0331 04:35:59.823185 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod 577fde9a3d5dd04bc97a1ab5909d3bc7cab38eef37d64292db74f72c418b6e74 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod 577fde9a3d5dd04bc97a1ab5909d3bc7cab38eef37d64292db74f72c418b6e74 network default NAD default], result "{}", err I0331 04:35:59.896595 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:35:59.896640 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod I0331 04:35:59.898024 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod, ips: 10.129.0.52 I0331 04:35:59.898055 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-verify-access-to-resources-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:01.024252 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-collect-data-pod] creating logical port default-tenant-managed_managed-f7ftt-collect-data-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:36:01.024392 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-collect-data-pod I0331 04:36:01.051720 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-collect-data-pod] pod update took 27.360347ms I0331 04:36:01.051741 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-collect-data-pod] addLogicalPort annotation time took 27.386388ms I0331 04:36:01.053142 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-collect-data-pod] addLogicalPort took 28.908296ms, libovsdb time 959.691µs I0331 04:36:01.396226 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-collect-data-pod 4972d33f524db76871dddb14af97965585cb7a04d8fdee97dff1284bbc0f39ba network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-collect-data-pod 4972d33f524db76871dddb14af97965585cb7a04d8fdee97dff1284bbc0f39ba network default NAD default] I0331 04:36:01.404580 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-collect-data-pod, hostIfaceName: 4972d33f524db76, network: default, NAD default, SandboxID: "4972d33f524db76871dddb14af97965585cb7a04d8fdee97dff1284bbc0f39ba", PCI device ID: , UID: "4c8e6d89-4eed-47d3-9c7a-dc4c471a8f2d", MAC: 0a:58:0a:81:00:36, IPs: [10.129.0.54/23] I0331 04:36:01.658998 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-collect-data-pod 4972d33f524db76871dddb14af97965585cb7a04d8fdee97dff1284bbc0f39ba network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-collect-data-pod 4972d33f524db76871dddb14af97965585cb7a04d8fdee97dff1284bbc0f39ba network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:93:0b:ec:07:2d\",\"name\":\"4972d33f524db76\"},{\"mac\":\"0a:58:0a:81:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5de94d01-3d39-49df-ac84-6d06c5cf8755\"}],\"ips\":[{\"address\":\"10.129.0.54/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:36:15.585104 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0331 04:36:18.600100 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0331 04:36:22.908411 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-collect-data-pod 4972d33f524db76871dddb14af97965585cb7a04d8fdee97dff1284bbc0f39ba network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-collect-data-pod 4972d33f524db76871dddb14af97965585cb7a04d8fdee97dff1284bbc0f39ba network default NAD default] I0331 04:36:23.011108 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-collect-data-pod 4972d33f524db76871dddb14af97965585cb7a04d8fdee97dff1284bbc0f39ba network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-collect-data-pod 4972d33f524db76871dddb14af97965585cb7a04d8fdee97dff1284bbc0f39ba network default NAD default], result "{}", err I0331 04:36:23.165113 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-collect-data-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:23.165240 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-collect-data-pod I0331 04:36:23.167218 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-collect-data-pod, ips: 10.129.0.54 I0331 04:36:23.167256 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-collect-data-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:23.608561 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-collect-task-params-pod] creating logical port default-tenant-managed_managed-f7ftt-collect-task-params-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:36:23.608736 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-collect-task-params-pod I0331 04:36:23.628281 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-collect-task-params-pod] pod update took 19.578502ms I0331 04:36:23.628300 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-collect-task-params-pod] addLogicalPort annotation time took 19.60395ms I0331 04:36:23.629691 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-collect-task-params-pod] addLogicalPort took 21.149829ms, libovsdb time 916.466µs I0331 04:36:24.012878 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-collect-task-params-pod d44a2cbf73eaf0d59c12e06938a8165d381ae00d2a3569d45295fb9942dbe22e network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-collect-task-params-pod d44a2cbf73eaf0d59c12e06938a8165d381ae00d2a3569d45295fb9942dbe22e network default NAD default] I0331 04:36:24.021783 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-collect-task-params-pod, hostIfaceName: d44a2cbf73eaf0d, network: default, NAD default, SandboxID: "d44a2cbf73eaf0d59c12e06938a8165d381ae00d2a3569d45295fb9942dbe22e", PCI device ID: , UID: "3ccedaff-4d8c-4b8f-b529-67605c87a033", MAC: 0a:58:0a:81:00:37, IPs: [10.129.0.55/23] I0331 04:36:24.073970 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-collect-task-params-pod d44a2cbf73eaf0d59c12e06938a8165d381ae00d2a3569d45295fb9942dbe22e network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-collect-task-params-pod d44a2cbf73eaf0d59c12e06938a8165d381ae00d2a3569d45295fb9942dbe22e network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:27:dc:29:44:14\",\"name\":\"d44a2cbf73eaf0d\"},{\"mac\":\"0a:58:0a:81:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fbd2f533-3e2c-4a8a-9687-2ed6dfc221ec\"}],\"ips\":[{\"address\":\"10.129.0.55/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:36:24.898433 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod] creating logical port default-tenant-managed_managed-f7ftt-collect-registry-token-secret-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:36:24.898596 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod I0331 04:36:24.919703 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod] pod update took 21.143546ms I0331 04:36:24.919727 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod] addLogicalPort annotation time took 21.173184ms I0331 04:36:24.921140 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod] addLogicalPort took 22.722272ms, libovsdb time 902.153µs I0331 04:36:25.281537 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod 2f6f87819771c6793a09074bd9c9a498ac207dc8e3b7fd603dcc1347c04f3241 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod 2f6f87819771c6793a09074bd9c9a498ac207dc8e3b7fd603dcc1347c04f3241 network default NAD default] I0331 04:36:25.291396 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-collect-registry-token-secret-pod, hostIfaceName: 2f6f87819771c67, network: default, NAD default, SandboxID: "2f6f87819771c6793a09074bd9c9a498ac207dc8e3b7fd603dcc1347c04f3241", PCI device ID: , UID: "d471a142-a33d-4002-84ca-c122bb7fbec5", MAC: 0a:58:0a:81:00:38, IPs: [10.129.0.56/23] I0331 04:36:25.548006 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod 2f6f87819771c6793a09074bd9c9a498ac207dc8e3b7fd603dcc1347c04f3241 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod 2f6f87819771c6793a09074bd9c9a498ac207dc8e3b7fd603dcc1347c04f3241 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:dd:2a:f2:4a:10\",\"name\":\"2f6f87819771c67\"},{\"mac\":\"0a:58:0a:81:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c2992f67-18c4-4bfe-b86b-c18343dd2764\"}],\"ips\":[{\"address\":\"10.129.0.56/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:36:26.262975 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-check-data-keys-pod] creating logical port default-tenant-managed_managed-f7ftt-check-data-keys-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:36:26.263102 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-check-data-keys-pod I0331 04:36:26.283991 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-check-data-keys-pod] pod update took 20.920897ms I0331 04:36:26.284014 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-check-data-keys-pod] addLogicalPort annotation time took 20.952097ms I0331 04:36:26.285539 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-check-data-keys-pod] addLogicalPort took 22.586892ms, libovsdb time 1.019228ms I0331 04:36:26.750686 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-check-data-keys-pod 6a71bf19127cd7aa6b47e87df2e5a7c8b0ee6c91b66305226038b685b2598abc network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-check-data-keys-pod 6a71bf19127cd7aa6b47e87df2e5a7c8b0ee6c91b66305226038b685b2598abc network default NAD default] I0331 04:36:26.760436 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-check-data-keys-pod, hostIfaceName: 6a71bf19127cd7a, network: default, NAD default, SandboxID: "6a71bf19127cd7aa6b47e87df2e5a7c8b0ee6c91b66305226038b685b2598abc", PCI device ID: , UID: "5eb6a284-531d-4dc5-9e06-fc3c3d051e62", MAC: 0a:58:0a:81:00:39, IPs: [10.129.0.57/23] I0331 04:36:26.781137 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-reduce-snapshot-pod] creating logical port default-tenant-managed_managed-f7ftt-reduce-snapshot-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:36:26.781311 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-reduce-snapshot-pod I0331 04:36:26.805165 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod] pod update took 23.889027ms I0331 04:36:26.805186 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-reduce-snapshot-pod] addLogicalPort annotation time took 23.913935ms I0331 04:36:26.806592 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod] addLogicalPort took 25.47275ms, libovsdb time 919.032µs I0331 04:36:26.811587 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-check-data-keys-pod 6a71bf19127cd7aa6b47e87df2e5a7c8b0ee6c91b66305226038b685b2598abc network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-check-data-keys-pod 6a71bf19127cd7aa6b47e87df2e5a7c8b0ee6c91b66305226038b685b2598abc network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:75:39:ce:d5:e8\",\"name\":\"6a71bf19127cd7a\"},{\"mac\":\"0a:58:0a:81:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c5270b18-aefb-4f31-8c37-546fc497279e\"}],\"ips\":[{\"address\":\"10.129.0.57/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:36:27.224303 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod c555b4a36f4602c3e49c291d2f19e6635b55224b215c6f1afc25fff9638fa1bd network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod c555b4a36f4602c3e49c291d2f19e6635b55224b215c6f1afc25fff9638fa1bd network default NAD default] I0331 04:36:27.233312 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-reduce-snapshot-pod, hostIfaceName: c555b4a36f4602c, network: default, NAD default, SandboxID: "c555b4a36f4602c3e49c291d2f19e6635b55224b215c6f1afc25fff9638fa1bd", PCI device ID: , UID: "c4d23629-eb35-4ac5-9e01-3a52abc39816", MAC: 0a:58:0a:81:00:3a, IPs: [10.129.0.58/23] I0331 04:36:27.488513 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod c555b4a36f4602c3e49c291d2f19e6635b55224b215c6f1afc25fff9638fa1bd network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod c555b4a36f4602c3e49c291d2f19e6635b55224b215c6f1afc25fff9638fa1bd network default NAD default], result "{\"interfaces\":[{\"mac\":\"c2:f8:81:3d:ae:96\",\"name\":\"c555b4a36f4602c\"},{\"mac\":\"0a:58:0a:81:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/df2696c3-6031-442d-9448-0242b1da664c\"}],\"ips\":[{\"address\":\"10.129.0.58/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:36:29.614936 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0331 04:36:31.008118 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-collect-task-params-pod d44a2cbf73eaf0d59c12e06938a8165d381ae00d2a3569d45295fb9942dbe22e network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-collect-task-params-pod d44a2cbf73eaf0d59c12e06938a8165d381ae00d2a3569d45295fb9942dbe22e network default NAD default] I0331 04:36:31.105289 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-collect-task-params-pod d44a2cbf73eaf0d59c12e06938a8165d381ae00d2a3569d45295fb9942dbe22e network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-collect-task-params-pod d44a2cbf73eaf0d59c12e06938a8165d381ae00d2a3569d45295fb9942dbe22e network default NAD default], result "{}", err I0331 04:36:31.184679 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-collect-task-params-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:31.184713 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-collect-task-params-pod I0331 04:36:31.186068 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-collect-task-params-pod, ips: 10.129.0.55 I0331 04:36:31.186093 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-collect-task-params-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:32.023684 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod 2f6f87819771c6793a09074bd9c9a498ac207dc8e3b7fd603dcc1347c04f3241 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod 2f6f87819771c6793a09074bd9c9a498ac207dc8e3b7fd603dcc1347c04f3241 network default NAD default] I0331 04:36:32.131230 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod 2f6f87819771c6793a09074bd9c9a498ac207dc8e3b7fd603dcc1347c04f3241 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod 2f6f87819771c6793a09074bd9c9a498ac207dc8e3b7fd603dcc1347c04f3241 network default NAD default], result "{}", err I0331 04:36:32.208394 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:32.208427 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod I0331 04:36:32.210012 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod, ips: 10.129.0.56 I0331 04:36:32.210048 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-collect-registry-token-secret-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:47.162800 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-check-data-keys-pod 6a71bf19127cd7aa6b47e87df2e5a7c8b0ee6c91b66305226038b685b2598abc network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-check-data-keys-pod 6a71bf19127cd7aa6b47e87df2e5a7c8b0ee6c91b66305226038b685b2598abc network default NAD default] I0331 04:36:47.257850 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-check-data-keys-pod 6a71bf19127cd7aa6b47e87df2e5a7c8b0ee6c91b66305226038b685b2598abc network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-check-data-keys-pod 6a71bf19127cd7aa6b47e87df2e5a7c8b0ee6c91b66305226038b685b2598abc network default NAD default], result "{}", err I0331 04:36:47.337471 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-check-data-keys-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:47.337519 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-check-data-keys-pod I0331 04:36:47.338974 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-check-data-keys-pod, ips: 10.129.0.57 I0331 04:36:47.339001 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-check-data-keys-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:56.237224 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod c555b4a36f4602c3e49c291d2f19e6635b55224b215c6f1afc25fff9638fa1bd network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod c555b4a36f4602c3e49c291d2f19e6635b55224b215c6f1afc25fff9638fa1bd network default NAD default] I0331 04:36:56.323858 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod c555b4a36f4602c3e49c291d2f19e6635b55224b215c6f1afc25fff9638fa1bd network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-reduce-snapshot-pod c555b4a36f4602c3e49c291d2f19e6635b55224b215c6f1afc25fff9638fa1bd network default NAD default], result "{}", err I0331 04:36:56.404505 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-reduce-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:56.404544 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-reduce-snapshot-pod I0331 04:36:56.406047 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-reduce-snapshot-pod, ips: 10.129.0.58 I0331 04:36:56.406075 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-reduce-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:36:57.361268 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-apply-mapping-pod] creating logical port default-tenant-managed_managed-f7ftt-apply-mapping-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:36:57.361396 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-apply-mapping-pod I0331 04:36:57.392360 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-apply-mapping-pod] pod update took 30.999639ms I0331 04:36:57.392383 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-apply-mapping-pod] addLogicalPort annotation time took 31.027597ms I0331 04:36:57.393823 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-apply-mapping-pod] addLogicalPort took 32.57695ms, libovsdb time 933.343µs I0331 04:36:57.732788 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-apply-mapping-pod 62a1b2f08a7d4d5c32a68459cd7ad1bf18157770d808531551bc7bfcc21eeab4 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-apply-mapping-pod 62a1b2f08a7d4d5c32a68459cd7ad1bf18157770d808531551bc7bfcc21eeab4 network default NAD default] I0331 04:36:57.741744 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-apply-mapping-pod, hostIfaceName: 62a1b2f08a7d4d5, network: default, NAD default, SandboxID: "62a1b2f08a7d4d5c32a68459cd7ad1bf18157770d808531551bc7bfcc21eeab4", PCI device ID: , UID: "23515341-9a31-4562-970b-f4fff1b9f56f", MAC: 0a:58:0a:81:00:3b, IPs: [10.129.0.59/23] I0331 04:36:57.792012 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-apply-mapping-pod 62a1b2f08a7d4d5c32a68459cd7ad1bf18157770d808531551bc7bfcc21eeab4 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-apply-mapping-pod 62a1b2f08a7d4d5c32a68459cd7ad1bf18157770d808531551bc7bfcc21eeab4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:4c:62:7d:83:f5\",\"name\":\"62a1b2f08a7d4d5\"},{\"mac\":\"0a:58:0a:81:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dc5c6889-73c1-4b46-9503-2c407c53fea6\"}],\"ips\":[{\"address\":\"10.129.0.59/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:37:18.434028 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-apply-mapping-pod 62a1b2f08a7d4d5c32a68459cd7ad1bf18157770d808531551bc7bfcc21eeab4 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-apply-mapping-pod 62a1b2f08a7d4d5c32a68459cd7ad1bf18157770d808531551bc7bfcc21eeab4 network default NAD default] I0331 04:37:18.500712 3934 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=9 I0331 04:37:18.546533 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-apply-mapping-pod 62a1b2f08a7d4d5c32a68459cd7ad1bf18157770d808531551bc7bfcc21eeab4 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-apply-mapping-pod 62a1b2f08a7d4d5c32a68459cd7ad1bf18157770d808531551bc7bfcc21eeab4 network default NAD default], result "{}", err I0331 04:37:18.639352 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-apply-mapping-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:37:18.639390 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-apply-mapping-pod I0331 04:37:18.640858 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-apply-mapping-pod, ips: 10.129.0.59 I0331 04:37:18.640914 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-apply-mapping-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:37:19.469134 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-filter-already-released-images-pod] creating logical port default-tenant-managed_managed-f7ftt-filter-already-released-images-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:37:19.469280 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-filter-already-released-images-pod I0331 04:37:19.491828 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod] pod update took 22.581842ms I0331 04:37:19.491855 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-filter-already-released-images-pod] addLogicalPort annotation time took 22.61029ms I0331 04:37:19.493391 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod] addLogicalPort took 24.272589ms, libovsdb time 1.056755ms I0331 04:37:19.841540 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod cb8bd4b4b583d26c9adff138d2fe1ea0d2e31c4d39c83b38c4a3de1d4329efa8 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod cb8bd4b4b583d26c9adff138d2fe1ea0d2e31c4d39c83b38c4a3de1d4329efa8 network default NAD default] I0331 04:37:19.850964 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-filter-already-released-images-pod, hostIfaceName: cb8bd4b4b583d26, network: default, NAD default, SandboxID: "cb8bd4b4b583d26c9adff138d2fe1ea0d2e31c4d39c83b38c4a3de1d4329efa8", PCI device ID: , UID: "ebd23812-99c6-4548-a64e-9fcd45a4ff1d", MAC: 0a:58:0a:81:00:3c, IPs: [10.129.0.60/23] I0331 04:37:20.105887 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod cb8bd4b4b583d26c9adff138d2fe1ea0d2e31c4d39c83b38c4a3de1d4329efa8 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod cb8bd4b4b583d26c9adff138d2fe1ea0d2e31c4d39c83b38c4a3de1d4329efa8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:49:4b:0c:36:13\",\"name\":\"cb8bd4b4b583d26\"},{\"mac\":\"0a:58:0a:81:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/07234090-5f9b-4b56-b5f5-5b2edd4e4104\"}],\"ips\":[{\"address\":\"10.129.0.60/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:37:38.606618 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod cb8bd4b4b583d26c9adff138d2fe1ea0d2e31c4d39c83b38c4a3de1d4329efa8 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod cb8bd4b4b583d26c9adff138d2fe1ea0d2e31c4d39c83b38c4a3de1d4329efa8 network default NAD default] I0331 04:37:38.702041 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod cb8bd4b4b583d26c9adff138d2fe1ea0d2e31c4d39c83b38c4a3de1d4329efa8 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-filter-already-released-images-pod cb8bd4b4b583d26c9adff138d2fe1ea0d2e31c4d39c83b38c4a3de1d4329efa8 network default NAD default], result "{}", err I0331 04:37:38.782951 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-filter-already-released-images-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:37:38.782987 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-filter-already-released-images-pod I0331 04:37:38.784571 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-filter-already-released-images-pod, ips: 10.129.0.60 I0331 04:37:38.784599 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-filter-already-released-images-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:37:39.515457 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-verify-conforma-pod] creating logical port default-tenant-managed_managed-f7ftt-verify-conforma-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:37:39.515595 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-verify-conforma-pod I0331 04:37:39.555409 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-verify-conforma-pod] pod update took 39.849187ms I0331 04:37:39.555432 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-verify-conforma-pod] addLogicalPort annotation time took 39.87713ms I0331 04:37:39.556798 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-verify-conforma-pod] addLogicalPort took 41.351245ms, libovsdb time 982.825µs I0331 04:37:39.886038 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-verify-conforma-pod 7f10d77b1af5dbf6bf3489dab93f1b554d7027652bd67d0aaf5b485852eab192 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-verify-conforma-pod 7f10d77b1af5dbf6bf3489dab93f1b554d7027652bd67d0aaf5b485852eab192 network default NAD default] I0331 04:37:39.894518 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-verify-conforma-pod, hostIfaceName: 7f10d77b1af5dbf, network: default, NAD default, SandboxID: "7f10d77b1af5dbf6bf3489dab93f1b554d7027652bd67d0aaf5b485852eab192", PCI device ID: , UID: "2229eeb8-c0d0-4577-ae52-01c2757ff196", MAC: 0a:58:0a:81:00:3d, IPs: [10.129.0.61/23] I0331 04:37:39.942657 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-verify-conforma-pod 7f10d77b1af5dbf6bf3489dab93f1b554d7027652bd67d0aaf5b485852eab192 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-verify-conforma-pod 7f10d77b1af5dbf6bf3489dab93f1b554d7027652bd67d0aaf5b485852eab192 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:e5:f8:af:aa:a7\",\"name\":\"7f10d77b1af5dbf\"},{\"mac\":\"0a:58:0a:81:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/01c230c3-92d1-4220-a054-a06b37939351\"}],\"ips\":[{\"address\":\"10.129.0.61/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:38:11.955099 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-verify-conforma-pod 7f10d77b1af5dbf6bf3489dab93f1b554d7027652bd67d0aaf5b485852eab192 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-verify-conforma-pod 7f10d77b1af5dbf6bf3489dab93f1b554d7027652bd67d0aaf5b485852eab192 network default NAD default] I0331 04:38:12.068462 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-verify-conforma-pod 7f10d77b1af5dbf6bf3489dab93f1b554d7027652bd67d0aaf5b485852eab192 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-verify-conforma-pod 7f10d77b1af5dbf6bf3489dab93f1b554d7027652bd67d0aaf5b485852eab192 network default NAD default], result "{}", err I0331 04:38:12.209673 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-verify-conforma-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:38:12.209739 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-verify-conforma-pod I0331 04:38:12.211519 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-verify-conforma-pod, ips: 10.129.0.61 I0331 04:38:12.211551 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-verify-conforma-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:38:13.069549 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-push-snapshot-pod] creating logical port default-tenant-managed_managed-f7ftt-push-snapshot-pod for pod on switch ip-10-0-0-29.ec2.internal I0331 04:38:13.069683 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-push-snapshot-pod I0331 04:38:13.098599 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-push-snapshot-pod] pod update took 28.943403ms I0331 04:38:13.098687 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-push-snapshot-pod] addLogicalPort annotation time took 29.036459ms I0331 04:38:13.100384 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-push-snapshot-pod] addLogicalPort took 30.848705ms, libovsdb time 1.09308ms I0331 04:38:13.445874 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-push-snapshot-pod 882687a8c401afdbddabf9b1a7934164e825b03fac9425aee373e90c1a759faa network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod 882687a8c401afdbddabf9b1a7934164e825b03fac9425aee373e90c1a759faa network default NAD default] I0331 04:38:13.454624 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-push-snapshot-pod, hostIfaceName: 882687a8c401afd, network: default, NAD default, SandboxID: "882687a8c401afdbddabf9b1a7934164e825b03fac9425aee373e90c1a759faa", PCI device ID: , UID: "6bfd068c-9bd8-4863-93fb-ecb23125dfdb", MAC: 0a:58:0a:81:00:3e, IPs: [10.129.0.62/23] I0331 04:38:13.502760 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-push-snapshot-pod 882687a8c401afdbddabf9b1a7934164e825b03fac9425aee373e90c1a759faa network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod 882687a8c401afdbddabf9b1a7934164e825b03fac9425aee373e90c1a759faa network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:04:a4:7a:3c:54\",\"name\":\"882687a8c401afd\"},{\"mac\":\"0a:58:0a:81:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bea7ece0-71e8-4e1e-938c-6222ba65979b\"}],\"ips\":[{\"address\":\"10.129.0.62/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:38:24.072131 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-push-snapshot-pod 882687a8c401afdbddabf9b1a7934164e825b03fac9425aee373e90c1a759faa network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod 882687a8c401afdbddabf9b1a7934164e825b03fac9425aee373e90c1a759faa network default NAD default] I0331 04:38:24.164162 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-push-snapshot-pod 882687a8c401afdbddabf9b1a7934164e825b03fac9425aee373e90c1a759faa network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod 882687a8c401afdbddabf9b1a7934164e825b03fac9425aee373e90c1a759faa network default NAD default], result "{}", err I0331 04:38:24.243060 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:38:24.243094 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-push-snapshot-pod I0331 04:38:24.244512 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-push-snapshot-pod, ips: 10.129.0.62 I0331 04:38:24.244540 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:38:41.708826 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2] creating logical port default-tenant-managed_managed-f7ftt-push-snapshot-pod-retry2 for pod on switch ip-10-0-0-29.ec2.internal I0331 04:38:41.708998 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 I0331 04:38:41.734800 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2] pod update took 25.835096ms I0331 04:38:41.734827 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2] addLogicalPort annotation time took 25.861695ms I0331 04:38:41.736292 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2] addLogicalPort took 27.48261ms, libovsdb time 1.004288ms I0331 04:38:42.078441 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 952bec6b435b9b4b0ef934b34fed715dcf00151afbe822f4936d7d888fc918a9 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 952bec6b435b9b4b0ef934b34fed715dcf00151afbe822f4936d7d888fc918a9 network default NAD default] I0331 04:38:42.087128 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-push-snapshot-pod-retry2, hostIfaceName: 952bec6b435b9b4, network: default, NAD default, SandboxID: "952bec6b435b9b4b0ef934b34fed715dcf00151afbe822f4936d7d888fc918a9", PCI device ID: , UID: "d23ba9af-9b7e-4209-bf7f-810f94c9bd67", MAC: 0a:58:0a:81:00:3f, IPs: [10.129.0.63/23] I0331 04:38:42.135871 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 952bec6b435b9b4b0ef934b34fed715dcf00151afbe822f4936d7d888fc918a9 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 952bec6b435b9b4b0ef934b34fed715dcf00151afbe822f4936d7d888fc918a9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:11:cd:79:52:fe\",\"name\":\"952bec6b435b9b4\"},{\"mac\":\"0a:58:0a:81:00:3f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d4d6c0d7-a454-4c10-9343-b3dc3d4b9c73\"}],\"ips\":[{\"address\":\"10.129.0.63/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:38:42.554721 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:38:42.555297 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:38:50.312188 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 952bec6b435b9b4b0ef934b34fed715dcf00151afbe822f4936d7d888fc918a9 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 952bec6b435b9b4b0ef934b34fed715dcf00151afbe822f4936d7d888fc918a9 network default NAD default] I0331 04:38:50.401381 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 952bec6b435b9b4b0ef934b34fed715dcf00151afbe822f4936d7d888fc918a9 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 952bec6b435b9b4b0ef934b34fed715dcf00151afbe822f4936d7d888fc918a9 network default NAD default], result "{}", err I0331 04:38:50.485749 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:38:50.485799 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 I0331 04:38:50.487310 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2, ips: 10.129.0.63 I0331 04:38:50.487344 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:38:57.018745 3934 base_network_controller_pods.go:487] [default/default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4] creating logical port default-tenant-managed_managed-f7ftt-push-snapshot-pod-retry4 for pod on switch ip-10-0-0-29.ec2.internal I0331 04:38:57.018939 3934 kube.go:256] Updating pod default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 I0331 04:38:57.046585 3934 pod.go:62] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4] pod update took 27.709677ms I0331 04:38:57.046608 3934 base_network_controller_pods.go:937] [default/default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4] addLogicalPort annotation time took 27.740636ms I0331 04:38:57.047906 3934 pods.go:271] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4] addLogicalPort took 29.16042ms, libovsdb time 878.46µs I0331 04:38:57.389670 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 94a56f94beee6d5a3052d69d8c3b2555f9a68e458175ca7cb856dbbbb0fac3f4 network default NAD default] ADD starting CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 94a56f94beee6d5a3052d69d8c3b2555f9a68e458175ca7cb856dbbbb0fac3f4 network default NAD default] I0331 04:38:57.398235 3934 helper_linux.go:450] ConfigureOVS: namespace: default-tenant-managed, podName: managed-f7ftt-push-snapshot-pod-retry4, hostIfaceName: 94a56f94beee6d5, network: default, NAD default, SandboxID: "94a56f94beee6d5a3052d69d8c3b2555f9a68e458175ca7cb856dbbbb0fac3f4", PCI device ID: , UID: "825f21ab-18b5-4009-98d9-ceb184807f83", MAC: 0a:58:0a:81:00:40, IPs: [10.129.0.64/23] I0331 04:38:57.447485 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 94a56f94beee6d5a3052d69d8c3b2555f9a68e458175ca7cb856dbbbb0fac3f4 network default NAD default] ADD finished CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 94a56f94beee6d5a3052d69d8c3b2555f9a68e458175ca7cb856dbbbb0fac3f4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:6b:b1:ab:0c:28\",\"name\":\"94a56f94beee6d5\"},{\"mac\":\"0a:58:0a:81:00:40\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/797d87ea-0226-46e7-8f5b-26a0a5509155\"}],\"ips\":[{\"address\":\"10.129.0.64/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0331 04:38:57.683553 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry3 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:38:57.684135 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry3 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:39:05.463052 3934 cni.go:403] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 94a56f94beee6d5a3052d69d8c3b2555f9a68e458175ca7cb856dbbbb0fac3f4 network default NAD default] DEL starting CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 94a56f94beee6d5a3052d69d8c3b2555f9a68e458175ca7cb856dbbbb0fac3f4 network default NAD default] I0331 04:39:05.558223 3934 cni.go:424] [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 94a56f94beee6d5a3052d69d8c3b2555f9a68e458175ca7cb856dbbbb0fac3f4 network default NAD default] DEL finished CNI request [default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 94a56f94beee6d5a3052d69d8c3b2555f9a68e458175ca7cb856dbbbb0fac3f4 network default NAD default], result "{}", err I0331 04:39:05.637470 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:39:05.637571 3934 pods.go:176] Deleting pod: default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 I0331 04:39:05.639549 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4, ips: 10.129.0.64 I0331 04:39:05.639584 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:39:12.033464 3934 cni.go:403] [default-tenant-managed/affinity-assistant-b07ee9c152-0 a4841ce4b88194a1891ae32c62137b87403bb25492cd0b56e1d6879ee0e588a0 network default NAD default] DEL starting CNI request [default-tenant-managed/affinity-assistant-b07ee9c152-0 a4841ce4b88194a1891ae32c62137b87403bb25492cd0b56e1d6879ee0e588a0 network default NAD default] I0331 04:39:12.128808 3934 cni.go:424] [default-tenant-managed/affinity-assistant-b07ee9c152-0 a4841ce4b88194a1891ae32c62137b87403bb25492cd0b56e1d6879ee0e588a0 network default NAD default] DEL finished CNI request [default-tenant-managed/affinity-assistant-b07ee9c152-0 a4841ce4b88194a1891ae32c62137b87403bb25492cd0b56e1d6879ee0e588a0 network default NAD default], result "{}", err I0331 04:39:12.183214 3934 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-b07ee9c152-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:39:12.183252 3934 pods.go:176] Deleting pod: default-tenant-managed/affinity-assistant-b07ee9c152-0 I0331 04:39:12.184507 3934 pods.go:236] Attempting to release IPs for pod: default-tenant-managed/affinity-assistant-b07ee9c152-0, ips: 10.129.0.53 I0331 04:39:12.184540 3934 obj_retry.go:464] Detected object default-tenant-managed/affinity-assistant-b07ee9c152-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:39:12.519957 3934 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-b07ee9c152-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:39:12.519978 3934 obj_retry.go:458] Detected object default-tenant-managed/affinity-assistant-b07ee9c152-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0331 04:39:12.527830 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod default-tenant-managed/affinity-assistant-b07ee9c152-0 I0331 04:39:12.527862 3934 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod default-tenant-managed/affinity-assistant-b07ee9c152-0 I0331 04:39:12.878252 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0331 04:39:12.878814 3934 obj_retry.go:464] Detected object default-tenant-managed/managed-f7ftt-push-snapshot-pod-retry5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0331 04:39:34.618287 3934 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9