+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-173.ec2.internal ++ [[ -n ip-10-0-2-173.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-173.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0329 02:51:26.393812428 - disable conntrack on geneve port' I0329 02:51:26.393812428 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0329 02:51:26.410343511 - starting ovnkube-node + echo 'I0329 02:51:26.410343511 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-173.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-173.ec2.internal --init-node ip-10-0-2-173.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-173.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0329 02:51:26.448158 3885 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0329 02:51:26.448201 3885 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-457e589fce.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0329 02:51:26.450026 3885 kube.go:419] Waiting for certificate I0329 02:51:26.450067 3885 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0329 02:51:26.450106 3885 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0329 02:51:26.450784 3885 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0329 02:51:26.476165 3885 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0329 02:51:26.476179 3885 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0329 02:51:26.479016 3885 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0329 02:51:26.481627 3885 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-9wjxp" I0329 02:51:26.489479 3885 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-9wjxp" I0329 02:51:26.489555 3885 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0329 02:51:27.450332 3885 kube.go:426] Certificate found I0329 02:51:27.450860 3885 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0329 02:51:27.451757 3885 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0329 02:51:27.497108 3885 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-30 02:46:26 +0000 UTC" deadline="2026-03-29 22:23:22.298229327 +0000 UTC" I0329 02:51:27.497204 3885 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h31m54.801040625s" I0329 02:51:27.517275 3885 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0329 02:51:27.526795 3885 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0329 02:51:27.532836 3885 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0329 02:51:27.533004 3885 factory.go:531] Starting watch factory I0329 02:51:27.533267 3885 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.533312 3885 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.533431 3885 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.533525 3885 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.533598 3885 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.533675 3885 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.533682 3885 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.533700 3885 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.533836 3885 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.533901 3885 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.534871 3885 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.534889 3885 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.540586 3885 controller_manager.go:368] Starting the ovnkube controller I0329 02:51:27.540618 3885 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-173.ec2.internal I0329 02:51:27.540668 3885 controller_manager.go:393] NBDB zone sync took: 41.543µs I0329 02:51:27.540679 3885 factory.go:531] Starting watch factory I0329 02:51:27.548879 3885 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.549535 3885 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0329 02:51:27.551493 3885 ovn_northd.go:121] Registering OVN northd metrics I0329 02:51:27.551670 3885 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.551713 3885 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.552273 3885 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.555700 3885 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0329 02:51:27.555770 3885 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0329 02:51:27.556743 3885 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.556995 3885 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0329 02:51:27.571906 3885 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0329 02:51:27.638340 3885 factory.go:1890] *v1.Service informer cache synced successfully I0329 02:51:27.638372 3885 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0329 02:51:27.638378 3885 factory.go:1890] *v1.Pod informer cache synced successfully I0329 02:51:27.638383 3885 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0329 02:51:27.638386 3885 factory.go:1890] *v1.Namespace informer cache synced successfully I0329 02:51:27.638389 3885 factory.go:1890] *v1.Node informer cache synced successfully I0329 02:51:27.638480 3885 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:27.638489 3885 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:27.638496 3885 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:27.638509 3885 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:27.641251 3885 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:27.641275 3885 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0329 02:51:27.643538 3885 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0329 02:51:27.643549 3885 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0329 02:51:27.643595 3885 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.643607 3885 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.646151 3885 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.646164 3885 factory.go:1890] *v1.Service informer cache synced successfully I0329 02:51:27.646173 3885 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0329 02:51:27.646176 3885 factory.go:1890] *v1.Pod informer cache synced successfully I0329 02:51:27.646179 3885 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0329 02:51:27.646182 3885 factory.go:1890] *v1.Namespace informer cache synced successfully I0329 02:51:27.646185 3885 factory.go:1890] *v1.Node informer cache synced successfully I0329 02:51:27.649476 3885 factory.go:1890] *v1.EgressIP informer cache synced successfully I0329 02:51:27.649524 3885 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.649539 3885 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.651940 3885 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0329 02:51:27.651949 3885 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0329 02:51:27.652781 3885 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.654770 3885 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0329 02:51:27.654835 3885 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.654852 3885 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.657055 3885 factory.go:1890] *v1.EgressIP informer cache synced successfully I0329 02:51:27.657184 3885 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.660255 3885 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0329 02:51:27.660307 3885 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.660318 3885 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.662519 3885 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0329 02:51:27.662779 3885 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.665915 3885 factory.go:1890] *v1.EgressService informer cache synced successfully I0329 02:51:27.665978 3885 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.665991 3885 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.668108 3885 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0329 02:51:27.668396 3885 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.671554 3885 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0329 02:51:27.671736 3885 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0329 02:51:27.671749 3885 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0329 02:51:27.673958 3885 factory.go:1890] *v1.EgressService informer cache synced successfully I0329 02:51:27.674103 3885 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0329 02:51:27.677447 3885 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0329 02:51:27.677532 3885 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.677551 3885 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.677566 3885 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.677577 3885 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.679770 3885 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0329 02:51:27.679979 3885 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.680198 3885 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0329 02:51:27.683401 3885 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0329 02:51:27.683413 3885 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0329 02:51:27.683419 3885 factory.go:636] Watch Factory start up complete, took: 150.422364ms I0329 02:51:27.683507 3885 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0329 02:51:27.684803 3885 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0329 02:51:27.690555 3885 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0329 02:51:27.690565 3885 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0329 02:51:27.690572 3885 factory.go:636] Watch Factory start up complete, took: 149.891972ms I0329 02:51:27.690579 3885 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-2-173.ec2.internal" zone I0329 02:51:27.723639 3885 default_node_network_controller.go:709] Initializing the default node network controller I0329 02:51:27.766065 3885 udn_isolation.go:101] Starting UDN host isolation manager I0329 02:51:27.772301 3885 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0329 02:51:27.816679 3885 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0329 02:51:27.816792 3885 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0329 02:51:27.816805 3885 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0329 02:51:27.847422 3885 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0329 02:51:27.847484 3885 default_node_network_controller.go:808] Node ip-10-0-2-173.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0329 02:51:27.852121 3885 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.173"] k8s.ovn.org/zone-name:ip-10-0-2-173.ec2.internal] on node ip-10-0-2-173.ec2.internal I0329 02:51:27.871765 3885 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0329 02:51:27.876816 3885 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0329 02:51:27.876951 3885 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0329 02:51:27.877092 3885 gateway_init.go:263] Preparing Gateway I0329 02:51:27.877103 3885 gateway_shared_intf.go:1658] Creating new gateway I0329 02:51:27.941651 3885 controller_manager.go:427] Waiting for node in zone sync took: 251.061353ms I0329 02:51:27.946428 3885 iptables.go:108] Creating table: filter chain: FORWARD I0329 02:51:27.947322 3885 controller_manager.go:608] Switching to transit router for layer2 networks I0329 02:51:27.947335 3885 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-173.ec2.internal I0329 02:51:27.952238 3885 iptables.go:108] Creating table: filter chain: OUTPUT I0329 02:51:27.958088 3885 gateway_shared_intf.go:1787] Gateway Creation Complete I0329 02:51:27.958101 3885 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-173.ec2.internal","mac-address":"0e:ad:0c:03:49:4f","ip-addresses":["10.0.2.173/24"],"ip-address":"10.0.2.173/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:83fe52b6-cf38-48e7-850d-2d0f5a467eaf k8s.ovn.org/node-encap-ips:["10.0.2.173"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.173/24"} k8s.ovn.org/zone-name:ip-10-0-2-173.ec2.internal] on node ip-10-0-2-173.ec2.internal I0329 02:51:27.963702 3885 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0329 02:51:27.968969 3885 controller_manager.go:288] SCTP support detected in OVN I0329 02:51:27.969481 3885 services_controller.go:69] Creating services controller for network=default I0329 02:51:27.969581 3885 udn_enabled_svc.go:80] Waiting for service informer to sync I0329 02:51:27.969692 3885 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0329 02:51:27.969729 3885 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0329 02:51:27.969749 3885 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0329 02:51:27.969759 3885 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0329 02:51:27.969910 3885 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0329 02:51:27.969966 3885 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0329 02:51:27.969992 3885 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 44.711µs I0329 02:51:27.970002 3885 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 26.04µs I0329 02:51:27.970010 3885 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 44.606µs I0329 02:51:27.970016 3885 udn_enabled_svc.go:95] Performing full resync I0329 02:51:27.970018 3885 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0329 02:51:27.970119 3885 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0329 02:51:27.970133 3885 default_network_controller.go:321] Starting the default network controller I0329 02:51:27.970226 3885 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 9.938µs I0329 02:51:27.970357 3885 udn_enabled_svc.go:99] Waiting for handler to sync I0329 02:51:27.970367 3885 udn_enabled_svc.go:104] Starting worker I0329 02:51:27.977694 3885 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0329 02:51:27.977748 3885 repair.go:33] Syncing exgw routes took 41.42µs I0329 02:51:27.977765 3885 default_network_controller.go:407] Starting all the Watchers... I0329 02:51:27.977898 3885 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0329 02:51:27.977902 3885 namespace.go:98] [openshift-dns] adding namespace I0329 02:51:27.977898 3885 namespace.go:98] [openshift-authentication-operator] adding namespace I0329 02:51:27.977912 3885 namespace.go:98] [openshift-insights] adding namespace I0329 02:51:27.977916 3885 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0329 02:51:27.977898 3885 namespace.go:98] [kube-public] adding namespace I0329 02:51:27.977915 3885 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0329 02:51:27.977926 3885 namespace.go:98] [openshift-ingress-canary] adding namespace I0329 02:51:27.977931 3885 namespace.go:98] [openshift-authentication] adding namespace I0329 02:51:27.977930 3885 namespace.go:98] [openshift-machine-config-operator] adding namespace I0329 02:51:27.977940 3885 namespace.go:98] [openshift-etcd] adding namespace I0329 02:51:27.977941 3885 namespace.go:98] [openshift-host-network] adding namespace I0329 02:51:27.977951 3885 namespace.go:98] [openshift-must-gather-operator] adding namespace I0329 02:51:27.977941 3885 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0329 02:51:27.977964 3885 namespace.go:98] [openshift-backplane-mobb] adding namespace I0329 02:51:27.978379 3885 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 470.73µs I0329 02:51:27.978394 3885 namespace.go:98] [openshift-backplane-cse] adding namespace I0329 02:51:27.978833 3885 namespace.go:102] [openshift-backplane-cse] adding namespace took 433.985µs I0329 02:51:27.978849 3885 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0329 02:51:27.979327 3885 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 471.785µs I0329 02:51:27.979344 3885 namespace.go:98] [openshift-backplane-cee] adding namespace I0329 02:51:27.979768 3885 namespace.go:102] [openshift-dns] adding namespace took 1.853446ms I0329 02:51:27.979785 3885 namespace.go:98] [openshift-monitoring] adding namespace I0329 02:51:27.980209 3885 namespace.go:102] [openshift-authentication-operator] adding namespace took 2.2915ms I0329 02:51:27.980222 3885 namespace.go:98] [kube-system] adding namespace I0329 02:51:27.980622 3885 namespace.go:102] [openshift-insights] adding namespace took 2.700616ms I0329 02:51:27.980637 3885 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0329 02:51:27.981052 3885 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 3.126581ms I0329 02:51:27.981062 3885 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0329 02:51:27.981437 3885 namespace.go:102] [kube-public] adding namespace took 3.508854ms I0329 02:51:27.981446 3885 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0329 02:51:27.981881 3885 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 3.950547ms I0329 02:51:27.981892 3885 namespace.go:98] [openshift-console] adding namespace I0329 02:51:27.982274 3885 namespace.go:102] [openshift-ingress-canary] adding namespace took 4.338494ms I0329 02:51:27.982282 3885 namespace.go:98] [openshift-apiserver-operator] adding namespace I0329 02:51:27.982733 3885 namespace.go:102] [openshift-authentication] adding namespace took 4.792599ms I0329 02:51:27.982750 3885 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0329 02:51:27.983142 3885 namespace.go:102] [openshift-machine-config-operator] adding namespace took 5.199709ms I0329 02:51:27.983155 3885 namespace.go:98] [openshift-backplane-tam] adding namespace I0329 02:51:27.983593 3885 namespace.go:102] [openshift-etcd] adding namespace took 5.645198ms I0329 02:51:27.983608 3885 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0329 02:51:27.984062 3885 namespace.go:102] [openshift-host-network] adding namespace took 6.105384ms I0329 02:51:27.984076 3885 namespace.go:98] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] adding namespace I0329 02:51:27.984492 3885 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.535464ms I0329 02:51:27.984502 3885 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0329 02:51:27.984915 3885 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.95529ms I0329 02:51:27.984928 3885 namespace.go:98] [openshift-config] adding namespace I0329 02:51:27.985331 3885 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.359957ms I0329 02:51:27.985339 3885 namespace.go:98] [openshift-dns-operator] adding namespace I0329 02:51:27.985738 3885 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.38942ms I0329 02:51:27.985752 3885 namespace.go:98] [openshift-multus] adding namespace W0329 02:51:27.986121 3885 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-shxnh: pod kube-system/global-pull-secret-syncer-shxnh: no pod IPs found I0329 02:51:27.986167 3885 namespace.go:102] [openshift-monitoring] adding namespace took 6.377005ms I0329 02:51:27.986178 3885 namespace.go:98] [openshift-network-node-identity] adding namespace I0329 02:51:27.986564 3885 namespace.go:102] [kube-system] adding namespace took 6.335916ms I0329 02:51:27.986582 3885 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0329 02:51:27.987405 3885 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.763476ms I0329 02:51:27.987419 3885 namespace.go:98] [openshift-kube-scheduler] adding namespace I0329 02:51:27.987861 3885 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.794083ms I0329 02:51:27.987875 3885 namespace.go:98] [openshift-logging] adding namespace I0329 02:51:27.988321 3885 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.867918ms I0329 02:51:27.988343 3885 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0329 02:51:27.988971 3885 namespace.go:102] [openshift-console] adding namespace took 7.072974ms I0329 02:51:27.988987 3885 namespace.go:98] [openshift-network-console] adding namespace I0329 02:51:27.989351 3885 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.06378ms I0329 02:51:27.989365 3885 namespace.go:98] [openshift-node] adding namespace I0329 02:51:27.989797 3885 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.041356ms I0329 02:51:27.989810 3885 namespace.go:98] [openshift-ingress] adding namespace I0329 02:51:27.990166 3885 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.005154ms I0329 02:51:27.990177 3885 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0329 02:51:27.990522 3885 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.910412ms I0329 02:51:27.990532 3885 namespace.go:98] [openshift-marketplace] adding namespace I0329 02:51:27.990683 3885 default_node_network_controller.go:518] Node connection status = connected I0329 02:51:27.990963 3885 namespace.go:102] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] adding namespace took 6.881844ms I0329 02:51:27.990975 3885 namespace.go:98] [openshift-machine-api] adding namespace I0329 02:51:27.991355 3885 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.849211ms I0329 02:51:27.991367 3885 namespace.go:98] [kube-node-lease] adding namespace I0329 02:51:27.991750 3885 namespace.go:102] [openshift-config] adding namespace took 6.817566ms I0329 02:51:27.991762 3885 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace W0329 02:51:27.992088 3885 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-2jw6j: pod openshift-multus/network-metrics-daemon-2jw6j: no pod IPs found W0329 02:51:27.992099 3885 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-7hs79: pod openshift-multus/network-metrics-daemon-7hs79: no pod IPs found I0329 02:51:27.992132 3885 namespace.go:102] [openshift-dns-operator] adding namespace took 6.789358ms I0329 02:51:27.992142 3885 namespace.go:98] [openshift] adding namespace I0329 02:51:27.992656 3885 namespace.go:102] [openshift-multus] adding namespace took 6.899984ms I0329 02:51:27.993185 3885 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.002155ms I0329 02:51:27.993197 3885 namespace.go:98] [openshift-package-operator] adding namespace I0329 02:51:27.993636 3885 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.048855ms I0329 02:51:27.993646 3885 namespace.go:98] [openshift-operators] adding namespace I0329 02:51:27.994003 3885 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.579379ms I0329 02:51:27.994016 3885 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0329 02:51:27.994357 3885 namespace.go:102] [openshift-logging] adding namespace took 6.477032ms I0329 02:51:27.994368 3885 namespace.go:98] [openshift-service-ca-operator] adding namespace I0329 02:51:27.994868 3885 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.519699ms I0329 02:51:27.994881 3885 namespace.go:98] [openshift-route-controller-manager] adding namespace I0329 02:51:27.995245 3885 namespace.go:102] [openshift-network-console] adding namespace took 6.252782ms I0329 02:51:27.995256 3885 namespace.go:98] [openshift-apiserver] adding namespace I0329 02:51:27.995631 3885 namespace.go:102] [openshift-node] adding namespace took 6.261884ms I0329 02:51:27.995642 3885 namespace.go:98] [openshift-console-operator] adding namespace I0329 02:51:27.996158 3885 namespace.go:102] [openshift-ingress] adding namespace took 6.344091ms I0329 02:51:27.996166 3885 namespace.go:98] [dedicated-admin] adding namespace I0329 02:51:27.996632 3885 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.448377ms I0329 02:51:27.996646 3885 namespace.go:98] [default] adding namespace I0329 02:51:27.997116 3885 namespace.go:102] [openshift-marketplace] adding namespace took 6.580599ms I0329 02:51:27.997127 3885 namespace.go:98] [openshift-network-diagnostics] adding namespace I0329 02:51:27.997655 3885 namespace.go:102] [openshift-machine-api] adding namespace took 6.675346ms I0329 02:51:27.997664 3885 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0329 02:51:27.998094 3885 namespace.go:102] [kube-node-lease] adding namespace took 6.72346ms I0329 02:51:27.998106 3885 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0329 02:51:27.998538 3885 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.772422ms I0329 02:51:27.998548 3885 namespace.go:98] [openshift-controller-manager] adding namespace I0329 02:51:27.999048 3885 namespace.go:102] [openshift] adding namespace took 6.899322ms I0329 02:51:27.999064 3885 namespace.go:98] [openshift-backplane-srep] adding namespace I0329 02:51:27.999565 3885 namespace.go:102] [openshift-package-operator] adding namespace took 6.363313ms I0329 02:51:27.999577 3885 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0329 02:51:27.999961 3885 namespace.go:102] [openshift-operators] adding namespace took 6.310205ms I0329 02:51:27.999973 3885 namespace.go:98] [openshift-config-operator] adding namespace I0329 02:51:28.000363 3885 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.341015ms I0329 02:51:28.000376 3885 namespace.go:98] [openshift-image-registry] adding namespace I0329 02:51:28.001162 3885 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.790144ms I0329 02:51:28.001174 3885 namespace.go:98] [openshift-backplane] adding namespace I0329 02:51:28.001578 3885 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.691847ms I0329 02:51:28.001590 3885 namespace.go:98] [openshift-cluster-version] adding namespace I0329 02:51:28.002037 3885 namespace.go:102] [openshift-apiserver] adding namespace took 6.775325ms I0329 02:51:28.002048 3885 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0329 02:51:28.002496 3885 namespace.go:102] [openshift-console-operator] adding namespace took 6.849851ms I0329 02:51:28.002508 3885 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0329 02:51:28.002974 3885 namespace.go:102] [dedicated-admin] adding namespace took 6.796115ms I0329 02:51:28.002997 3885 namespace.go:98] [openshift-user-workload-monitoring] adding namespace W0329 02:51:28.003510 3885 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-zz6r9: pod openshift-network-diagnostics/network-check-target-zz6r9: no pod IPs found W0329 02:51:28.003523 3885 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-r6mfp: pod openshift-network-diagnostics/network-check-target-r6mfp: no pod IPs found I0329 02:51:28.003563 3885 namespace.go:102] [default] adding namespace took 6.912687ms I0329 02:51:28.003575 3885 namespace.go:98] [openshift-infra] adding namespace I0329 02:51:28.004191 3885 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.059743ms I0329 02:51:28.004200 3885 namespace.go:98] [openshift-console-user-settings] adding namespace I0329 02:51:28.004682 3885 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.012662ms I0329 02:51:28.004697 3885 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0329 02:51:28.005066 3885 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.95393ms I0329 02:51:28.005080 3885 namespace.go:98] [openshift-backplane-csm] adding namespace I0329 02:51:28.005488 3885 namespace.go:102] [openshift-controller-manager] adding namespace took 6.936286ms I0329 02:51:28.005500 3885 namespace.go:98] [openshift-config-managed] adding namespace I0329 02:51:28.005918 3885 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.847567ms I0329 02:51:28.005931 3885 namespace.go:98] [openshift-operators-redhat] adding namespace I0329 02:51:28.006279 3885 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.697945ms I0329 02:51:28.006691 3885 namespace.go:102] [openshift-config-operator] adding namespace took 6.714917ms I0329 02:51:28.006701 3885 namespace.go:98] [openshift-kube-apiserver] adding namespace I0329 02:51:28.007067 3885 namespace.go:102] [openshift-image-registry] adding namespace took 6.68632ms I0329 02:51:28.007427 3885 namespace.go:102] [openshift-backplane] adding namespace took 6.248445ms I0329 02:51:28.007437 3885 namespace.go:98] [openshift-customer-monitoring] adding namespace I0329 02:51:28.007813 3885 namespace.go:102] [openshift-cluster-version] adding namespace took 6.218391ms I0329 02:51:28.008160 3885 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.107767ms I0329 02:51:28.008171 3885 namespace.go:98] [openshift-network-operator] adding namespace I0329 02:51:28.008545 3885 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.032308ms I0329 02:51:28.008557 3885 namespace.go:98] [openshift-ingress-operator] adding namespace I0329 02:51:28.008910 3885 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 5.907527ms I0329 02:51:28.009299 3885 namespace.go:102] [openshift-infra] adding namespace took 5.719502ms I0329 02:51:28.009704 3885 namespace.go:102] [openshift-console-user-settings] adding namespace took 5.498234ms I0329 02:51:28.010062 3885 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.360972ms I0329 02:51:28.010428 3885 namespace.go:102] [openshift-backplane-csm] adding namespace took 5.342882ms I0329 02:51:28.010903 3885 namespace.go:102] [openshift-config-managed] adding namespace took 5.39864ms I0329 02:51:28.011277 3885 namespace.go:102] [openshift-operators-redhat] adding namespace took 5.341433ms I0329 02:51:28.011654 3885 namespace.go:102] [openshift-kube-apiserver] adding namespace took 4.948667ms I0329 02:51:28.012002 3885 namespace.go:102] [openshift-customer-monitoring] adding namespace took 4.56113ms I0329 02:51:28.012344 3885 namespace.go:102] [openshift-network-operator] adding namespace took 4.168631ms I0329 02:51:28.012676 3885 namespace.go:102] [openshift-ingress-operator] adding namespace took 4.114433ms I0329 02:51:28.012996 3885 zone_ic_handler.go:186] Time taken to create transit switch: 243.522µs I0329 02:51:28.013048 3885 master.go:532] Adding or Updating local node "ip-10-0-2-173.ec2.internal" for network "default" I0329 02:51:28.013126 3885 obj_retry.go:544] Failed to create *v1.Node ip-10-0-1-37.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-37.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-37.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-37.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-37.ec2.internal W0329 02:51:28.019473 3885 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-r6mfp: pod openshift-network-diagnostics/network-check-target-r6mfp: no pod IPs found W0329 02:51:28.019487 3885 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-2jw6j: pod openshift-multus/network-metrics-daemon-2jw6j: no pod IPs found W0329 02:51:28.019499 3885 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-shxnh: pod kube-system/global-pull-secret-syncer-shxnh: no pod IPs found I0329 02:51:28.020325 3885 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-2-173.ec2.internal for the network default I0329 02:51:28.020948 3885 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0329 02:51:28.020971 3885 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0329 02:51:28.020983 3885 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0329 02:51:28.021004 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-37.ec2.internal in network "default" I0329 02:51:28.021026 3885 node_tracker.go:235] Node ip-10-0-1-37.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-37.ec2.internal" I0329 02:51:28.021037 3885 node_tracker.go:169] Node ip-10-0-1-37.ec2.internal switch + router changed, syncing services in network "default" I0329 02:51:28.021044 3885 services_controller.go:574] Full service sync requested for network=default I0329 02:51:28.021051 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-173.ec2.internal in network "default" I0329 02:51:28.021074 3885 node_tracker.go:169] Node ip-10-0-2-173.ec2.internal switch + router changed, syncing services in network "default" I0329 02:51:28.021081 3885 services_controller.go:574] Full service sync requested for network=default I0329 02:51:28.121898 3885 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0329 02:51:28.121912 3885 services_controller.go:208] Setting up event handlers for services for network=default I0329 02:51:28.121962 3885 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0329 02:51:28.121994 3885 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0329 02:51:28.122005 3885 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0329 02:51:28.222493 3885 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0329 02:51:28.222509 3885 repair.go:60] Starting repairing loop for services I0329 02:51:28.222583 3885 repair.go:131] Deleted 0 stale service LBs I0329 02:51:28.222600 3885 repair.go:137] Deleted 0 stale Chassis Template Vars I0329 02:51:28.222614 3885 repair.go:62] Finished repairing loop for services: 106.459µs I0329 02:51:28.222638 3885 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0329 02:51:28.222647 3885 services_controller.go:254] Starting workers for network=default I0329 02:51:28.222983 3885 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-shxnh] creating logical port kube-system_global-pull-secret-syncer-shxnh for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:28.222993 3885 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-7hs79, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-7hs79: suppressed error logged: pod openshift-multus/network-metrics-daemon-7hs79: no pod IPs found I0329 02:51:28.223029 3885 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-zz6r9, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-zz6r9: suppressed error logged: pod openshift-network-diagnostics/network-check-target-zz6r9: no pod IPs found I0329 02:51:28.223060 3885 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-2jw6j] creating logical port openshift-multus_network-metrics-daemon-2jw6j for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:28.223034 3885 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-r6mfp] creating logical port openshift-network-diagnostics_network-check-target-r6mfp for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:28.223158 3885 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-shxnh I0329 02:51:28.223164 3885 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-2jw6j I0329 02:51:28.223163 3885 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-r6mfp I0329 02:51:28.245977 3885 pod.go:62] [openshift-multus/network-metrics-daemon-2jw6j] pod update took 22.849476ms I0329 02:51:28.245996 3885 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-2jw6j] addLogicalPort annotation time took 22.869529ms I0329 02:51:28.246909 3885 pods.go:271] [openshift-multus/network-metrics-daemon-2jw6j] addLogicalPort took 23.855807ms, libovsdb time 650.412µs I0329 02:51:28.252700 3885 pod.go:62] [kube-system/global-pull-secret-syncer-shxnh] pod update took 29.636883ms I0329 02:51:28.252720 3885 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-shxnh] addLogicalPort annotation time took 29.658918ms I0329 02:51:28.252805 3885 pod.go:62] [openshift-network-diagnostics/network-check-target-r6mfp] pod update took 29.670911ms I0329 02:51:28.252825 3885 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-r6mfp] addLogicalPort annotation time took 29.691041ms I0329 02:51:28.253777 3885 pods.go:271] [openshift-network-diagnostics/network-check-target-r6mfp] addLogicalPort took 30.753806ms, libovsdb time 720.537µs I0329 02:51:28.254105 3885 pods.go:271] [kube-system/global-pull-secret-syncer-shxnh] addLogicalPort took 31.140448ms, libovsdb time 1.006963ms I0329 02:51:28.254538 3885 admin_network_policy_controller.go:227] Starting controller default-network-controller I0329 02:51:28.254562 3885 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0329 02:51:28.254571 3885 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0329 02:51:28.254575 3885 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0329 02:51:28.254738 3885 repair.go:29] Repairing admin network policies took 158.04µs I0329 02:51:28.254833 3885 repair.go:92] Repairing baseline admin network policies took 87.702µs I0329 02:51:28.255304 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0329 02:51:28.255330 3885 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0329 02:51:28.257047 3885 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0329 02:51:28.257066 3885 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0329 02:51:28.257074 3885 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0329 02:51:28.258288 3885 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0329 02:51:28.258305 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0329 02:51:28.258314 3885 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0329 02:51:28.258805 3885 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0329 02:51:28.258826 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0329 02:51:28.258835 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0329 02:51:28.260846 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0329 02:51:28.260864 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0329 02:51:28.260871 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0329 02:51:28.262208 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0329 02:51:28.262231 3885 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0329 02:51:28.262242 3885 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0329 02:51:28.263841 3885 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0329 02:51:28.263857 3885 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0329 02:51:28.263867 3885 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0329 02:51:28.264826 3885 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0329 02:51:28.264846 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0329 02:51:28.264854 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0329 02:51:28.265364 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0329 02:51:28.265379 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0329 02:51:28.265387 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0329 02:51:28.266089 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0329 02:51:28.266109 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0329 02:51:28.266117 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0329 02:51:28.266622 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0329 02:51:28.266646 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0329 02:51:28.266653 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0329 02:51:28.267168 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0329 02:51:28.267182 3885 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0329 02:51:28.267191 3885 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0329 02:51:28.269368 3885 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0329 02:51:28.269389 3885 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0329 02:51:28.269397 3885 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0329 02:51:28.270322 3885 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0329 02:51:28.270342 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0329 02:51:28.270349 3885 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0329 02:51:28.270667 3885 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0329 02:51:28.270691 3885 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0329 02:51:28.270707 3885 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0329 02:51:28.271268 3885 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0329 02:51:28.271286 3885 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0329 02:51:28.271296 3885 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0329 02:51:28.272393 3885 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0329 02:51:28.272411 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0329 02:51:28.272418 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0329 02:51:28.272998 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0329 02:51:28.273021 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0329 02:51:28.273435 3885 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0329 02:51:28.273451 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0329 02:51:28.274304 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0329 02:51:28.274329 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0329 02:51:28.274338 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0329 02:51:28.275022 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0329 02:51:28.275040 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0329 02:51:28.275047 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0329 02:51:28.276724 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0329 02:51:28.276742 3885 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0329 02:51:28.276748 3885 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0329 02:51:28.277080 3885 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0329 02:51:28.277101 3885 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0329 02:51:28.277113 3885 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0329 02:51:28.278827 3885 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0329 02:51:28.278850 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0329 02:51:28.278861 3885 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0329 02:51:28.279365 3885 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0329 02:51:28.279383 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0329 02:51:28.279391 3885 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0329 02:51:28.279909 3885 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0329 02:51:28.279930 3885 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0329 02:51:28.279951 3885 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0329 02:51:28.280685 3885 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0329 02:51:28.280704 3885 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0329 02:51:28.280712 3885 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0329 02:51:28.281754 3885 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0329 02:51:28.281770 3885 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0329 02:51:28.281778 3885 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0329 02:51:28.282705 3885 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0329 02:51:28.282726 3885 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0329 02:51:28.282737 3885 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0329 02:51:28.284128 3885 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0329 02:51:28.284153 3885 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0329 02:51:28.284171 3885 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0329 02:51:28.285595 3885 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0329 02:51:28.290302 3885 egressfirewall.go:319] Starting EgressFirewall controller I0329 02:51:28.290329 3885 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0329 02:51:28.290351 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0329 02:51:28.290368 3885 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0329 02:51:28.290354 3885 controller.go:133] Adding controller egress-firewall-controller event handlers I0329 02:51:28.290380 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0329 02:51:28.290385 3885 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0329 02:51:28.290746 3885 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0329 02:51:28.290766 3885 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0329 02:51:28.290778 3885 egressqos.go:193] Setting up event handlers for EgressQoS I0329 02:51:28.290867 3885 egressfirewall.go:628] Syncing node "ip-10-0-1-37.ec2.internal" for egress firewall I0329 02:51:28.290888 3885 egressfirewall.go:628] Syncing node "ip-10-0-2-173.ec2.internal" for egress firewall I0329 02:51:28.290911 3885 egressqos.go:245] Starting EgressQoS Controller I0329 02:51:28.290928 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0329 02:51:28.290937 3885 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0329 02:51:28.290942 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0329 02:51:28.290946 3885 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0329 02:51:28.290951 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0329 02:51:28.290955 3885 shared_informer.go:356] "Caches are synced" controller="egressqos" I0329 02:51:28.290959 3885 egressqos.go:259] Repairing EgressQoSes I0329 02:51:28.290963 3885 egressqos.go:399] Starting repairing loop for egressqos I0329 02:51:28.291061 3885 egressqos.go:401] Finished repairing loop for egressqos: 97.747µs I0329 02:51:28.291084 3885 egressservice_zone.go:132] Setting up event handlers for Egress Services I0329 02:51:28.291116 3885 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-173.ec2.internal I0329 02:51:28.291185 3885 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-173.ec2.internal : 69.087µs I0329 02:51:28.291345 3885 egressservice_zone.go:210] Starting Egress Services Controller I0329 02:51:28.291357 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0329 02:51:28.291364 3885 shared_informer.go:356] "Caches are synced" controller="egressservices" I0329 02:51:28.291371 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0329 02:51:28.291375 3885 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0329 02:51:28.291382 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0329 02:51:28.291387 3885 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0329 02:51:28.291393 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0329 02:51:28.291397 3885 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0329 02:51:28.291401 3885 egressservice_zone.go:228] Repairing Egress Services I0329 02:51:28.291756 3885 master_controller.go:88] Starting Admin Policy Based Route Controller I0329 02:51:28.291765 3885 external_controller.go:278] Starting Admin Policy Based Route Controller I0329 02:51:28.291832 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-173.ec2.internal I0329 02:51:28.291845 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-173.ec2.internal: 17.387µs I0329 02:51:28.291852 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-37.ec2.internal I0329 02:51:28.291855 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-37.ec2.internal: 4.177µs I0329 02:51:28.291954 3885 default_network_controller.go:568] Completing all the Watchers took 314.175665ms I0329 02:51:28.291977 3885 default_network_controller.go:572] Starting unidling controllers I0329 02:51:28.291989 3885 unidle.go:46] Registering OVN SB ControllerEvent handler I0329 02:51:28.291999 3885 unidle.go:63] Populating Initial ContollerEvent events I0329 02:51:28.292014 3885 unidle.go:79] Setting up event handlers for services I0329 02:51:28.292146 3885 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0329 02:51:28.491622 3885 default_node_network_controller.go:518] Node connection status = connected I0329 02:51:28.992190 3885 default_node_network_controller.go:518] Node connection status = connected I0329 02:51:29.148499 3885 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0329 02:51:29.492253 3885 default_node_network_controller.go:518] Node connection status = connected I0329 02:51:29.507762 3885 bridgeconfig.go:537] Gateway is ready I0329 02:51:29.507781 3885 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0329 02:51:29.517800 3885 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.173. Updating OVN encap IP. I0329 02:51:29.522174 3885 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.173 - it is already configured I0329 02:51:29.522190 3885 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.173/24:{} 172.20.0.1/32:{}]. Updating annotations. I0329 02:51:29.522474 3885 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.173/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-173.ec2.internal","mac-address":"0e:ad:0c:03:49:4f","ip-addresses":["10.0.2.173/24"],"ip-address":"10.0.2.173/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:83fe52b6-cf38-48e7-850d-2d0f5a467eaf k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.173/24"}] on node ip-10-0-2-173.ec2.internal I0329 02:51:29.537414 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-173.ec2.internal in network "default" I0329 02:51:29.537487 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-173.ec2.internal I0329 02:51:29.537503 3885 node_tracker.go:169] Node ip-10-0-2-173.ec2.internal switch + router changed, syncing services in network "default" I0329 02:51:29.537506 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-173.ec2.internal: 47.13µs I0329 02:51:29.537505 3885 egressfirewall.go:628] Syncing node "ip-10-0-2-173.ec2.internal" for egress firewall I0329 02:51:29.537525 3885 services_controller.go:574] Full service sync requested for network=default I0329 02:51:29.537592 3885 master.go:532] Adding or Updating local node "ip-10-0-2-173.ec2.internal" for network "default" I0329 02:51:29.537690 3885 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.2.173 I0329 02:51:29.537803 3885 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-173.ec2.internal I0329 02:51:29.543537 3885 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-2-173.ec2.internal. Updating no re-route policies I0329 02:51:29.543609 3885 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-mhrm8 I0329 02:51:29.543620 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-zz6r9 I0329 02:51:29.543627 3885 obj_retry.go:364] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-mhrm8 I0329 02:51:29.543633 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-zz6r9 I0329 02:51:29.543630 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-dr542 I0329 02:51:29.543646 3885 obj_retry.go:384] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-mhrm8 after 0 failed attempt(s) I0329 02:51:29.543609 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qwjst I0329 02:51:29.543652 3885 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-shxnh I0329 02:51:29.543652 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-173.ec2.internal I0329 02:51:29.543672 3885 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-shxnh I0329 02:51:29.543674 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-2jw6j I0329 02:51:29.543680 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-dtz6s I0329 02:51:29.543680 3885 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-173.ec2.internal I0329 02:51:29.543686 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-2jw6j I0329 02:51:29.543688 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-dtz6s I0329 02:51:29.543690 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-dr542 I0329 02:51:29.543694 3885 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-173.ec2.internal I0329 02:51:29.543693 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-r6mfp I0329 02:51:29.543707 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-dtz6s after 0 failed attempt(s) I0329 02:51:29.543706 3885 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-zz6r9, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-zz6r9: suppressed error logged: pod openshift-network-diagnostics/network-check-target-zz6r9: no pod IPs found I0329 02:51:29.543665 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qwjst I0329 02:51:29.543725 3885 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-2jw6j] creating logical port openshift-multus_network-metrics-daemon-2jw6j for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:29.543727 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-4krqq I0329 02:51:29.543734 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-qwjst after 0 failed attempt(s) I0329 02:51:29.543682 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-173.ec2.internal I0329 02:51:29.543759 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-173.ec2.internal after 0 failed attempt(s) I0329 02:51:29.543727 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-8tmdw I0329 02:51:29.543780 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-8tmdw I0329 02:51:29.543795 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-8tmdw after 0 failed attempt(s) I0329 02:51:29.543733 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-r6mfp I0329 02:51:29.543817 3885 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-lhpw4 I0329 02:51:29.543827 3885 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-lhpw4 I0329 02:51:29.543829 3885 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-r6mfp] creating logical port openshift-network-diagnostics_network-check-target-r6mfp for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:29.543842 3885 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-lhpw4 after 0 failed attempt(s) I0329 02:51:29.543714 3885 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-shxnh] creating logical port kube-system_global-pull-secret-syncer-shxnh for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:29.543715 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-dr542 after 0 failed attempt(s) I0329 02:51:29.543715 3885 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-173.ec2.internal after 0 failed attempt(s) I0329 02:51:29.543696 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-tzcpl I0329 02:51:29.543980 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-tzcpl I0329 02:51:29.544001 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-tzcpl after 0 failed attempt(s) I0329 02:51:29.543634 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-5zlf6 I0329 02:51:29.544017 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-5zlf6 I0329 02:51:29.544034 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-5zlf6 after 0 failed attempt(s) I0329 02:51:29.543664 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-4crxm I0329 02:51:29.544050 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-4crxm I0329 02:51:29.544087 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-4crxm after 0 failed attempt(s) I0329 02:51:29.543741 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-4krqq I0329 02:51:29.544128 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-4krqq after 0 failed attempt(s) I0329 02:51:29.544553 3885 pods.go:271] [openshift-network-diagnostics/network-check-target-r6mfp] addLogicalPort took 728.418µs, libovsdb time 294.672µs I0329 02:51:29.544570 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-r6mfp after 0 failed attempt(s) I0329 02:51:29.544683 3885 pods.go:271] [openshift-multus/network-metrics-daemon-2jw6j] addLogicalPort took 972.91µs, libovsdb time 417.854µs I0329 02:51:29.544699 3885 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-2jw6j after 0 failed attempt(s) I0329 02:51:29.544796 3885 pods.go:271] [kube-system/global-pull-secret-syncer-shxnh] addLogicalPort took 1.092964ms, libovsdb time 486.978µs I0329 02:51:29.544807 3885 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-shxnh after 0 failed attempt(s) I0329 02:51:29.553321 3885 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0329 02:51:29.564414 3885 iptables.go:108] Creating table: mangle chain: OUTPUT I0329 02:51:29.567556 3885 iptables.go:108] Creating table: nat chain: OUTPUT I0329 02:51:29.596551 3885 iptables.go:108] Creating table: nat chain: PREROUTING I0329 02:51:29.641661 3885 iptables.go:108] Creating table: filter chain: FORWARD I0329 02:51:29.658199 3885 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0329 02:51:29.658299 3885 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0329 02:51:29.658339 3885 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0329 02:51:29.658444 3885 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0329 02:51:29.658521 3885 gateway_init.go:325] Gateway and management port readiness took 1.67050437s I0329 02:51:29.658552 3885 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0329 02:51:29.658589 3885 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0329 02:51:29.658601 3885 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0329 02:51:29.658610 3885 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0329 02:51:29.658670 3885 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0329 02:51:29.658754 3885 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 51.192µs I0329 02:51:29.659096 3885 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0329 02:51:29.659186 3885 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 57.168µs I0329 02:51:29.659204 3885 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 109.323µs I0329 02:51:29.659220 3885 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0329 02:51:29.659229 3885 nad_controller.go:173] [node-nad-controller NAD controller]: started I0329 02:51:29.659234 3885 default_node_network_controller.go:898] Starting the default node network controller I0329 02:51:29.659246 3885 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0329 02:51:29.659320 3885 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 14.043µs I0329 02:51:29.659547 3885 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0329 02:51:29.659558 3885 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0329 02:51:29.659584 3885 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a9f650 0xc0002d3c70 0xc0002d3ce0 0xc0002d3d50 0xc0002d3dc0 0xc0002d3e30 0xc0002d3ea0 0xc000aad810 0xc0002d3f10 0xc0002ec000 0xc0007fceb0 0xc00627d1f0 0xc0002ec070 map[0x2f67040:0xc002ba64e0 0x2f67580:0xc002ba6720 0x2f68000:0xc0010d8300 0x2f682a0:0xc002ba6480 0x2f687e0:0xc002ba6540 0x2f68a80:0xc002ba65a0 0x2f68d20:0xc002ba6600 0x2f69500:0xc002ba6780 0x2f697a0:0xc002ba67e0 0x2f93d40:0xc00188a0c0 0x2f940e0:0xc00188a1e0 0x2f94bc0:0xc000bb1920 0x2f96520:0xc001b0e000 0x2f97000:0xc00188a240 0x2f97e80:0xc001b0e060] 0xc000aa2690 0} I0329 02:51:29.659669 3885 gateway.go:151] Starting gateway service sync I0329 02:51:29.660182 3885 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0329 02:51:29.662617 3885 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0329 02:51:29.671746 3885 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0329 02:51:29.672935 3885 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0329 02:51:29.674191 3885 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0329 02:51:29.708912 3885 gateway.go:168] Gateway service sync done. Time taken: 49.230073ms I0329 02:51:29.709337 3885 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30239 I0329 02:51:29.709829 3885 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a9f650 0xc0002d3c70 0xc0002d3ce0 0xc0002d3d50 0xc0002d3dc0 0xc0002d3e30 0xc0002d3ea0 0xc000aad810 0xc0002d3f10 0xc0002ec000 0xc0007fceb0 0xc00627d1f0 0xc0002ec070 map[0x2f67040:0xc002ba64e0 0x2f67580:0xc002ba6720 0x2f68000:0xc0010d8300 0x2f682a0:0xc002ba6480 0x2f687e0:0xc002ba6540 0x2f68a80:0xc002ba65a0 0x2f68d20:0xc002ba6600 0x2f69500:0xc002ba6780 0x2f697a0:0xc002ba67e0 0x2f93d40:0xc00188a0c0 0x2f940e0:0xc00188a1e0 0x2f94bc0:0xc000bb1920 0x2f96520:0xc001b0e000 0x2f97000:0xc00188a240 0x2f97e80:0xc001b0e060] 0xc000aa2690 0} I0329 02:51:29.710389 3885 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000a9f650 0xc0002d3c70 0xc0002d3ce0 0xc0002d3d50 0xc0002d3dc0 0xc0002d3e30 0xc0002d3ea0 0xc000aad810 0xc0002d3f10 0xc0002ec000 0xc0007fceb0 0xc00627d1f0 0xc0002ec070 map[0x2f67040:0xc002ba64e0 0x2f67580:0xc002ba6720 0x2f68000:0xc0010d8300 0x2f682a0:0xc002ba6480 0x2f687e0:0xc002ba6540 0x2f68a80:0xc002ba65a0 0x2f68d20:0xc002ba6600 0x2f69500:0xc002ba6780 0x2f697a0:0xc002ba67e0 0x2f93d40:0xc00188a0c0 0x2f940e0:0xc00188a1e0 0x2f94bc0:0xc000bb1920 0x2f96520:0xc001b0e000 0x2f97000:0xc00188a240 0x2f97e80:0xc001b0e060] 0xc000aa2690 0} I0329 02:51:29.710430 3885 gateway.go:519] Reconciling gateway with updates I0329 02:51:29.716310 3885 gateway.go:326] Spawning Conntrack Rule Check Thread I0329 02:51:29.716383 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0329 02:51:29.716396 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0329 02:51:29.716398 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0329 02:51:29.716414 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0329 02:51:29.716415 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0329 02:51:29.716420 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0329 02:51:29.716428 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0329 02:51:29.716443 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0329 02:51:29.716453 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0329 02:51:29.716455 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0329 02:51:29.716458 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0329 02:51:29.716487 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0329 02:51:29.716493 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0329 02:51:29.716495 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0329 02:51:29.716501 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0329 02:51:29.716501 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0329 02:51:29.716512 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0329 02:51:29.716517 3885 default_node_network_controller.go:943] Gateway and management port readiness took 56.948954ms I0329 02:51:29.716518 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0329 02:51:29.716487 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0329 02:51:29.716530 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0329 02:51:29.716532 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0329 02:51:29.716546 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0329 02:51:29.716564 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0329 02:51:29.716591 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0329 02:51:29.716500 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0329 02:51:29.716604 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0329 02:51:29.716607 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0329 02:51:29.716608 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0329 02:51:29.716608 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:51:29.716619 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0329 02:51:29.716623 3885 port_claim.go:71] Svc openshift-ingress/router-default: port 30780 is already open, no action needed I0329 02:51:29.716624 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0329 02:51:29.716514 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0329 02:51:29.716633 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0329 02:51:29.716645 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0329 02:51:29.716643 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0329 02:51:29.716662 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0329 02:51:29.716667 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0329 02:51:29.716634 3885 port_claim.go:71] Svc openshift-ingress/router-default: port 30101 is already open, no action needed I0329 02:51:29.716522 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0329 02:51:29.716689 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0329 02:51:29.716537 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0329 02:51:29.716716 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0329 02:51:29.716724 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0329 02:51:29.716606 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0329 02:51:29.716753 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0329 02:51:29.716525 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0329 02:51:29.716767 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0329 02:51:29.716769 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0329 02:51:29.716774 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0329 02:51:29.716780 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0329 02:51:29.716786 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0329 02:51:29.716521 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0329 02:51:29.716799 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0329 02:51:29.716416 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0329 02:51:29.716571 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0329 02:51:29.716843 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0329 02:51:29.716857 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0329 02:51:29.716860 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0329 02:51:29.716522 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0329 02:51:29.716587 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0329 02:51:29.716589 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0329 02:51:29.716901 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0329 02:51:29.716572 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0329 02:51:29.716720 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0329 02:51:29.716590 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0329 02:51:29.716926 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0329 02:51:29.716587 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0329 02:51:29.716933 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0329 02:51:29.716942 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0329 02:51:29.716954 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0329 02:51:29.716966 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0329 02:51:29.716975 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0329 02:51:29.716986 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0329 02:51:29.716944 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0329 02:51:29.716589 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0329 02:51:29.717010 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0329 02:51:29.717013 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0329 02:51:29.717026 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0329 02:51:29.716592 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0329 02:51:29.717043 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0329 02:51:29.716594 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0329 02:51:29.717054 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0329 02:51:29.716975 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0329 02:51:29.716598 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0329 02:51:29.717092 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0329 02:51:29.717044 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0329 02:51:29.717114 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0329 02:51:29.716628 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:51:29.716600 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0329 02:51:29.716615 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0329 02:51:29.717145 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0329 02:51:29.716637 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0329 02:51:29.717186 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0329 02:51:29.716701 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0329 02:51:29.717198 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0329 02:51:29.716703 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0329 02:51:29.717258 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0329 02:51:29.716917 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0329 02:51:29.716917 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0329 02:51:29.717320 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0329 02:51:29.716931 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:29.717334 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:29.716592 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0329 02:51:29.716627 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0329 02:51:29.716581 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0329 02:51:29.717390 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0329 02:51:29.718182 3885 node_ip_handler_linux.go:160] Node IP manager is running W0329 02:51:29.756788 3885 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0329 02:51:29.756962 3885 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0329 02:51:29.757075 3885 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0329 02:51:29.766419 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0329 02:51:29.766434 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0329 02:51:29.766424 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0329 02:51:29.766452 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0329 02:51:29.766459 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0329 02:51:29.766485 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0329 02:51:29.766493 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0329 02:51:29.766531 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0329 02:51:29.766544 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0329 02:51:29.766553 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0329 02:51:29.806722 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0329 02:51:29.806746 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0329 02:51:29.806753 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0329 02:51:29.806758 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0329 02:51:29.806765 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0329 02:51:29.806774 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0329 02:51:29.845560 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0329 02:51:29.845587 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0329 02:51:29.845594 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0329 02:51:29.845602 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0329 02:51:29.845619 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0329 02:51:29.845627 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0329 02:51:29.845633 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0329 02:51:29.845640 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0329 02:51:29.845659 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0329 02:51:29.845682 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0329 02:51:29.845691 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0329 02:51:31.173136 3885 default_node_network_controller.go:1364] Starting node controller node sync W0329 02:51:31.173139 3885 egressip_healthcheck.go:76] Health checking using insecure connection I0329 02:51:31.173244 3885 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0329 02:51:31.184866 3885 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 11.709832ms I0329 02:51:31.184995 3885 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0329 02:51:31.185256 3885 egressservice_node.go:93] Setting up event handlers for Egress Services I0329 02:51:31.185384 3885 egressservice_node.go:183] Starting Egress Services Controller I0329 02:51:31.185401 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0329 02:51:31.185414 3885 shared_informer.go:356] "Caches are synced" controller="egressservices" I0329 02:51:31.185420 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0329 02:51:31.185425 3885 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0329 02:51:31.185431 3885 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0329 02:51:31.185435 3885 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0329 02:51:31.185439 3885 egressservice_node.go:197] Repairing Egress Services I0329 02:51:31.194481 3885 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0329 02:51:31.194494 3885 external_controller.go:278] Starting Admin Policy Based Route Controller I0329 02:51:31.194711 3885 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0329 02:51:31.194724 3885 link_network_manager.go:117] Link manager is running I0329 02:51:31.194735 3885 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0329 02:51:31.194818 3885 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0329 02:51:31.195198 3885 vrf_manager.go:128] VRF manager is running I0329 02:51:31.195323 3885 node_controller_manager.go:437] Removing flows to drop GARP I0329 02:51:31.195336 3885 gateway.go:519] Reconciling gateway with updates I0329 02:51:31.219295 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0329 02:51:31.219308 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0329 02:51:31.219317 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0329 02:51:31.219317 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:51:31.219329 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0329 02:51:31.219330 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:51:31.219336 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0329 02:51:31.219346 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0329 02:51:31.219355 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0329 02:51:31.219362 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0329 02:51:31.219366 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0329 02:51:31.219380 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0329 02:51:31.219384 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0329 02:51:31.219392 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0329 02:51:31.219396 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0329 02:51:31.219398 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0329 02:51:31.219404 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0329 02:51:31.219409 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0329 02:51:31.219414 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0329 02:51:31.219414 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0329 02:51:31.219420 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0329 02:51:31.219368 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0329 02:51:31.219385 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0329 02:51:31.219419 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0329 02:51:31.219432 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0329 02:51:31.219440 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0329 02:51:31.219402 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:31.219450 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0329 02:51:31.219454 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0329 02:51:31.219459 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:31.219391 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0329 02:51:31.219484 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0329 02:51:31.219453 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0329 02:51:31.219488 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0329 02:51:31.219489 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0329 02:51:31.219499 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0329 02:51:31.219499 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0329 02:51:31.219383 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0329 02:51:31.219510 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0329 02:51:31.219510 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0329 02:51:31.219507 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0329 02:51:31.219511 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0329 02:51:31.219516 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0329 02:51:31.219522 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0329 02:51:31.219524 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0329 02:51:31.219520 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0329 02:51:31.219528 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0329 02:51:31.219531 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0329 02:51:31.219533 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0329 02:51:31.219532 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0329 02:51:31.219440 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0329 02:51:31.219543 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0329 02:51:31.219384 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0329 02:51:31.219575 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0329 02:51:31.219502 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0329 02:51:31.219458 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0329 02:51:31.219477 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0329 02:51:31.219433 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0329 02:51:31.219454 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0329 02:51:31.219606 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0329 02:51:31.219610 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0329 02:51:31.219613 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0329 02:51:31.219614 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0329 02:51:31.219627 3885 port_claim.go:71] Svc openshift-ingress/router-default: port 30780 is already open, no action needed I0329 02:51:31.219629 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0329 02:51:31.219636 3885 port_claim.go:71] Svc openshift-ingress/router-default: port 30101 is already open, no action needed I0329 02:51:31.219642 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0329 02:51:31.219641 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0329 02:51:31.219648 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0329 02:51:31.219486 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0329 02:51:31.219666 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0329 02:51:31.219498 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0329 02:51:31.219678 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0329 02:51:31.219492 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0329 02:51:31.219514 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0329 02:51:31.219694 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0329 02:51:31.219515 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0329 02:51:31.219513 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0329 02:51:31.219719 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0329 02:51:31.219515 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0329 02:51:31.219727 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0329 02:51:31.219525 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0329 02:51:31.219735 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0329 02:51:31.219737 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0329 02:51:31.219728 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0329 02:51:31.219745 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0329 02:51:31.219749 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0329 02:51:31.219429 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0329 02:51:31.219753 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0329 02:51:31.219758 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0329 02:51:31.219668 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0329 02:51:31.219767 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0329 02:51:31.219770 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0329 02:51:31.219503 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0329 02:51:31.219799 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0329 02:51:31.219684 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0329 02:51:31.219683 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0329 02:51:31.219751 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0329 02:51:31.219843 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0329 02:51:31.219857 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0329 02:51:31.219786 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0329 02:51:31.219889 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0329 02:51:31.219743 3885 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0329 02:51:31.219921 3885 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0329 02:51:31.220038 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0329 02:51:31.220061 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0329 02:51:31.220072 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0329 02:51:31.257352 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0329 02:51:31.257380 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0329 02:51:31.257394 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0329 02:51:31.257402 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0329 02:51:31.257410 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0329 02:51:31.257417 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0329 02:51:31.257437 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0329 02:51:31.257477 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0329 02:51:31.257498 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0329 02:51:31.257511 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0329 02:51:31.257531 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0329 02:51:31.298953 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0329 02:51:31.298970 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0329 02:51:31.337788 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0329 02:51:31.337812 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0329 02:51:31.337822 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0329 02:51:31.337832 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0329 02:51:31.337850 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0329 02:51:31.337872 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0329 02:51:31.337882 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0329 02:51:31.337890 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0329 02:51:31.337898 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0329 02:51:31.337904 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0329 02:51:31.337911 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0329 02:51:31.337923 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0329 02:51:31.337940 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0329 02:51:31.337945 3885 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0329 02:51:33.362713 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-173.ec2.internal I0329 02:51:33.362747 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-173.ec2.internal: 39.4µs I0329 02:51:33.391933 3885 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7c558b4c9b-8rqfz] creating logical port openshift-image-registry_image-registry-7c558b4c9b-8rqfz for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:33.392099 3885 kube.go:256] Updating pod openshift-image-registry/image-registry-7c558b4c9b-8rqfz I0329 02:51:33.413118 3885 pod.go:62] [openshift-image-registry/image-registry-7c558b4c9b-8rqfz] pod update took 21.068029ms I0329 02:51:33.413141 3885 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-7c558b4c9b-8rqfz] addLogicalPort annotation time took 21.092663ms I0329 02:51:33.414456 3885 pods.go:271] [openshift-image-registry/image-registry-7c558b4c9b-8rqfz] addLogicalPort took 22.537586ms, libovsdb time 1.018295ms I0329 02:51:33.435855 3885 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-wp5xk] creating logical port openshift-dns_dns-default-wp5xk for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:33.436085 3885 kube.go:256] Updating pod openshift-dns/dns-default-wp5xk I0329 02:51:33.442111 3885 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-vsqts] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-vsqts for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:33.442231 3885 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-vsqts I0329 02:51:33.443070 3885 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29579210-c26s7 for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:33.443152 3885 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-b5btm] creating logical port openshift-ingress-canary_ingress-canary-b5btm for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:33.443165 3885 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 I0329 02:51:33.443273 3885 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-b5btm I0329 02:51:33.453630 3885 pod.go:62] [openshift-dns/dns-default-wp5xk] pod update took 17.675042ms I0329 02:51:33.453656 3885 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-wp5xk] addLogicalPort annotation time took 17.71132ms I0329 02:51:33.454730 3885 pods.go:271] [openshift-dns/dns-default-wp5xk] addLogicalPort took 18.918436ms, libovsdb time 767.482µs I0329 02:51:33.462065 3885 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vsqts] pod update took 19.86183ms I0329 02:51:33.462086 3885 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-vsqts] addLogicalPort annotation time took 19.88657ms I0329 02:51:33.463525 3885 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vsqts] addLogicalPort took 21.427418ms, libovsdb time 1.193134ms I0329 02:51:33.464117 3885 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7] pod update took 20.969456ms I0329 02:51:33.464139 3885 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7] addLogicalPort annotation time took 20.992765ms I0329 02:51:33.465186 3885 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7] addLogicalPort took 22.125433ms, libovsdb time 738.812µs I0329 02:51:33.467144 3885 pod.go:62] [openshift-ingress-canary/ingress-canary-b5btm] pod update took 23.893851ms I0329 02:51:33.467162 3885 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-b5btm] addLogicalPort annotation time took 23.91351ms I0329 02:51:33.468066 3885 pods.go:271] [openshift-ingress-canary/ingress-canary-b5btm] addLogicalPort took 24.928651ms, libovsdb time 703.104µs I0329 02:51:33.800850 3885 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vsqts 9d73abc09796fe0c2b9a9748ac3a2912af45dea7c20da4c3dab77ad514b5e48d network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vsqts 9d73abc09796fe0c2b9a9748ac3a2912af45dea7c20da4c3dab77ad514b5e48d network default NAD default] I0329 02:51:33.826910 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-vsqts, hostIfaceName: 9d73abc09796fe0, network: default, NAD default, SandboxID: "9d73abc09796fe0c2b9a9748ac3a2912af45dea7c20da4c3dab77ad514b5e48d", PCI device ID: , UID: "39293e01-1a92-4abe-b420-cb318b554346", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0329 02:51:33.837118 3885 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 3213a5c2796137c4633b43fdf23ae105a4a0f3028664ed2d03d28d699f1077ac network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 3213a5c2796137c4633b43fdf23ae105a4a0f3028664ed2d03d28d699f1077ac network default NAD default] I0329 02:51:33.838902 3885 cni.go:403] [openshift-ingress-canary/ingress-canary-b5btm b31164a8848281fb732ae14bd47d8c649c7a458a99c245e897f66c0229d21b60 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-b5btm b31164a8848281fb732ae14bd47d8c649c7a458a99c245e897f66c0229d21b60 network default NAD default] I0329 02:51:33.847074 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29579210-c26s7, hostIfaceName: 3213a5c2796137c, network: default, NAD default, SandboxID: "3213a5c2796137c4633b43fdf23ae105a4a0f3028664ed2d03d28d699f1077ac", PCI device ID: , UID: "fa9eb832-b052-4052-8540-0b443e8b92e3", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0329 02:51:33.847998 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-b5btm, hostIfaceName: b31164a8848281f, network: default, NAD default, SandboxID: "b31164a8848281fb732ae14bd47d8c649c7a458a99c245e897f66c0229d21b60", PCI device ID: , UID: "f194088b-8638-48c1-ac68-3c3d8d4114a7", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0329 02:51:33.885816 3885 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vsqts 9d73abc09796fe0c2b9a9748ac3a2912af45dea7c20da4c3dab77ad514b5e48d network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-vsqts 9d73abc09796fe0c2b9a9748ac3a2912af45dea7c20da4c3dab77ad514b5e48d network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:52:3f:bd:49:34\",\"name\":\"9d73abc09796fe0\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8c18e60-caa0-4087-ad57-9d8b4fe96d6e\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:33.895085 3885 cni.go:424] [openshift-ingress-canary/ingress-canary-b5btm b31164a8848281fb732ae14bd47d8c649c7a458a99c245e897f66c0229d21b60 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-b5btm b31164a8848281fb732ae14bd47d8c649c7a458a99c245e897f66c0229d21b60 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:fd:c5:2e:64:28\",\"name\":\"b31164a8848281f\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ad08eb3c-6b6b-4fe3-a2b5-a57bcd064b94\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:33.895573 3885 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 3213a5c2796137c4633b43fdf23ae105a4a0f3028664ed2d03d28d699f1077ac network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 3213a5c2796137c4633b43fdf23ae105a4a0f3028664ed2d03d28d699f1077ac network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:49:75:7e:5e:90\",\"name\":\"3213a5c2796137c\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/58a9f6ab-e464-436e-a913-81ebb7487549\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:35.627532 3885 default_network_controller.go:1008] Node "ip-10-0-1-37.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:51:36.852299 3885 cni.go:403] [openshift-network-diagnostics/network-check-target-r6mfp acedb038984989a6711db482b511beac6ea46b778deb4da741de60dd937eb634 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-r6mfp acedb038984989a6711db482b511beac6ea46b778deb4da741de60dd937eb634 network default NAD default] I0329 02:51:36.861656 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-r6mfp, hostIfaceName: acedb038984989a, network: default, NAD default, SandboxID: "acedb038984989a6711db482b511beac6ea46b778deb4da741de60dd937eb634", PCI device ID: , UID: "2acd4193-708a-4b6c-9e8f-9e58f2b8eb4e", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0329 02:51:36.901576 3885 cni.go:424] [openshift-network-diagnostics/network-check-target-r6mfp acedb038984989a6711db482b511beac6ea46b778deb4da741de60dd937eb634 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-r6mfp acedb038984989a6711db482b511beac6ea46b778deb4da741de60dd937eb634 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:7b:9c:d5:c6:5a\",\"name\":\"acedb038984989a\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d934faaa-99e3-438d-b650-632997aa54e7\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 02:51:37.660788 3885 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:40.210880 3885 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-mhrm8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:51:40.210924 3885 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-mhrm8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0329 02:51:40.650649 3885 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0329 02:51:40.676917 3885 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-mhrm8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:51:40.676934 3885 obj_retry.go:458] Detected object kube-system/ecr-credential-refresh-initial-mhrm8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:51:41.752607 3885 cni.go:403] [kube-system/global-pull-secret-syncer-shxnh 5f45990658dd3c0bebaf76115ca8a7ec2f2ecfdb3f5db599c9541fca5d9c7b44 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-shxnh 5f45990658dd3c0bebaf76115ca8a7ec2f2ecfdb3f5db599c9541fca5d9c7b44 network default NAD default] I0329 02:51:41.762398 3885 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-shxnh, hostIfaceName: 5f45990658dd3c0, network: default, NAD default, SandboxID: "5f45990658dd3c0bebaf76115ca8a7ec2f2ecfdb3f5db599c9541fca5d9c7b44", PCI device ID: , UID: "05ee5f2a-9555-4f31-a4a9-131b1f1f27b5", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0329 02:51:41.802139 3885 cni.go:424] [kube-system/global-pull-secret-syncer-shxnh 5f45990658dd3c0bebaf76115ca8a7ec2f2ecfdb3f5db599c9541fca5d9c7b44 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-shxnh 5f45990658dd3c0bebaf76115ca8a7ec2f2ecfdb3f5db599c9541fca5d9c7b44 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:69:85:b4:71:1c\",\"name\":\"5f45990658dd3c0\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe024395-fcae-47d1-956f-4d2616366ab7\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:44.006218 3885 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-8777bd9bb-9wdd5] creating logical port openshift-console-operator_console-operator-8777bd9bb-9wdd5 for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.006334 3885 kube.go:256] Updating pod openshift-console-operator/console-operator-8777bd9bb-9wdd5 I0329 02:51:44.013527 3885 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-t2dxv] creating logical port openshift-service-ca-operator_service-ca-operator-7b89bc695f-t2dxv for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.013595 3885 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-7b89bc695f-t2dxv I0329 02:51:44.027045 3885 pod.go:62] [openshift-console-operator/console-operator-8777bd9bb-9wdd5] pod update took 20.736342ms I0329 02:51:44.027064 3885 base_network_controller_pods.go:937] [default/openshift-console-operator/console-operator-8777bd9bb-9wdd5] addLogicalPort annotation time took 20.760291ms I0329 02:51:44.028167 3885 pods.go:271] [openshift-console-operator/console-operator-8777bd9bb-9wdd5] addLogicalPort took 21.9646ms, libovsdb time 847.923µs I0329 02:51:44.034033 3885 pod.go:62] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t2dxv] pod update took 20.456067ms I0329 02:51:44.034049 3885 base_network_controller_pods.go:937] [default/openshift-service-ca-operator/service-ca-operator-7b89bc695f-t2dxv] addLogicalPort annotation time took 20.475263ms I0329 02:51:44.034994 3885 pods.go:271] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t2dxv] addLogicalPort took 21.480195ms, libovsdb time 705.616µs I0329 02:51:44.118953 3885 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-9wxht] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-555cc59f77-9wxht for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.119002 3885 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-n6krz] creating logical port openshift-monitoring_cluster-monitoring-operator-6dd897695f-n6krz for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.119069 3885 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-9wxht I0329 02:51:44.119071 3885 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-6dd897695f-n6krz I0329 02:51:44.127320 3885 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.127377 3885 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn I0329 02:51:44.147203 3885 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-9wxht] pod update took 28.191425ms I0329 02:51:44.147227 3885 base_network_controller_pods.go:937] [default/openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-9wxht] addLogicalPort annotation time took 28.213341ms I0329 02:51:44.147314 3885 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-n6krz] pod update took 28.266915ms I0329 02:51:44.147335 3885 base_network_controller_pods.go:937] [default/openshift-monitoring/cluster-monitoring-operator-6dd897695f-n6krz] addLogicalPort annotation time took 28.289576ms I0329 02:51:44.148387 3885 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-9wxht] addLogicalPort took 29.443473ms, libovsdb time 822.161µs I0329 02:51:44.148663 3885 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-n6krz] addLogicalPort took 29.670939ms, libovsdb time 1.031397ms I0329 02:51:44.149335 3885 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7bf7699dd9-knnqb] creating logical port openshift-ingress_router-default-7bf7699dd9-knnqb for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.149394 3885 kube.go:256] Updating pod openshift-ingress/router-default-7bf7699dd9-knnqb I0329 02:51:44.154904 3885 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn] pod update took 27.541811ms I0329 02:51:44.154924 3885 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn] addLogicalPort annotation time took 27.56125ms I0329 02:51:44.155872 3885 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn] addLogicalPort took 28.564303ms, libovsdb time 695.3µs I0329 02:51:44.155911 3885 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-88thf] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67b99c68bb-88thf for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.155958 3885 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-88thf I0329 02:51:44.174989 3885 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-84747c8b65-sn66l] creating logical port openshift-ingress_router-default-84747c8b65-sn66l for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.175149 3885 kube.go:256] Updating pod openshift-ingress/router-default-84747c8b65-sn66l I0329 02:51:44.186651 3885 pod.go:62] [openshift-ingress/router-default-7bf7699dd9-knnqb] pod update took 37.275841ms I0329 02:51:44.186677 3885 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-7bf7699dd9-knnqb] addLogicalPort annotation time took 37.300857ms I0329 02:51:44.186680 3885 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-88thf] pod update took 30.733261ms I0329 02:51:44.186698 3885 base_network_controller_pods.go:937] [default/openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-88thf] addLogicalPort annotation time took 30.752731ms I0329 02:51:44.187842 3885 pods.go:271] [openshift-ingress/router-default-7bf7699dd9-knnqb] addLogicalPort took 38.510293ms, libovsdb time 840.353µs I0329 02:51:44.188127 3885 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-88thf] addLogicalPort took 32.218948ms, libovsdb time 1.037756ms I0329 02:51:44.201319 3885 pod.go:62] [openshift-ingress/router-default-84747c8b65-sn66l] pod update took 26.20013ms I0329 02:51:44.201340 3885 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-84747c8b65-sn66l] addLogicalPort annotation time took 26.219812ms I0329 02:51:44.202356 3885 pods.go:271] [openshift-ingress/router-default-84747c8b65-sn66l] addLogicalPort took 27.376994ms, libovsdb time 689.754µs I0329 02:51:44.222511 3885 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-6fwbw] creating logical port openshift-network-diagnostics_network-check-source-5c76bd6f9b-6fwbw for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.222595 3885 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-5c76bd6f9b-6fwbw I0329 02:51:44.237334 3885 pod.go:62] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-6fwbw] pod update took 14.755835ms I0329 02:51:44.237352 3885 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-source-5c76bd6f9b-6fwbw] addLogicalPort annotation time took 14.776232ms I0329 02:51:44.238393 3885 pods.go:271] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-6fwbw] addLogicalPort took 15.891396ms, libovsdb time 762.085µs I0329 02:51:44.310798 3885 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-5577457d66-z26lm] creating logical port openshift-insights_insights-operator-5577457d66-z26lm for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.310884 3885 kube.go:256] Updating pod openshift-insights/insights-operator-5577457d66-z26lm I0329 02:51:44.319712 3885 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-x5tgn] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-x5tgn for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.319816 3885 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-x5tgn I0329 02:51:44.331922 3885 pod.go:62] [openshift-insights/insights-operator-5577457d66-z26lm] pod update took 21.057839ms I0329 02:51:44.331948 3885 base_network_controller_pods.go:937] [default/openshift-insights/insights-operator-5577457d66-z26lm] addLogicalPort annotation time took 21.087685ms I0329 02:51:44.333361 3885 pods.go:271] [openshift-insights/insights-operator-5577457d66-z26lm] addLogicalPort took 22.57435ms, libovsdb time 1.04691ms I0329 02:51:44.340373 3885 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-x5tgn] pod update took 20.574866ms I0329 02:51:44.340394 3885 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-x5tgn] addLogicalPort annotation time took 20.598284ms I0329 02:51:44.341789 3885 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-x5tgn] addLogicalPort took 22.090368ms, libovsdb time 992.6µs I0329 02:51:44.341851 3885 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-58d8f6745c-7zxcs] creating logical port openshift-network-console_networking-console-plugin-58d8f6745c-7zxcs for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:44.341929 3885 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-58d8f6745c-7zxcs I0329 02:51:44.365611 3885 pod.go:62] [openshift-network-console/networking-console-plugin-58d8f6745c-7zxcs] pod update took 23.697146ms I0329 02:51:44.365684 3885 base_network_controller_pods.go:937] [default/openshift-network-console/networking-console-plugin-58d8f6745c-7zxcs] addLogicalPort annotation time took 23.778636ms I0329 02:51:44.366825 3885 pods.go:271] [openshift-network-console/networking-console-plugin-58d8f6745c-7zxcs] addLogicalPort took 24.985351ms, libovsdb time 742.254µs W0329 02:51:44.680585 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0329 02:51:44.958041 3885 default_network_controller.go:1008] Node "ip-10-0-1-37.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:51:45.294032 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-37.ec2.internal I0329 02:51:45.294060 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-37.ec2.internal: 31.97µs I0329 02:51:45.294067 3885 default_network_controller.go:1008] Node "ip-10-0-1-37.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:51:45.294140 3885 egressfirewall.go:628] Syncing node "ip-10-0-1-37.ec2.internal" for egress firewall I0329 02:51:45.643878 3885 cni.go:403] [openshift-console-operator/console-operator-8777bd9bb-9wdd5 3a67aae0cb7daa906b418c42e8c58ef90e8aa4a5afd20587fca5be5016e313df network default NAD default] ADD starting CNI request [openshift-console-operator/console-operator-8777bd9bb-9wdd5 3a67aae0cb7daa906b418c42e8c58ef90e8aa4a5afd20587fca5be5016e313df network default NAD default] I0329 02:51:45.647168 3885 cni.go:403] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn ffb6c07435bb4dc139ed6b43b70615c60d90958ca9b36ea34436e28b50ee6ba6 network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn ffb6c07435bb4dc139ed6b43b70615c60d90958ca9b36ea34436e28b50ee6ba6 network default NAD default] I0329 02:51:45.647754 3885 cni.go:403] [openshift-insights/insights-operator-5577457d66-z26lm 02e267c654c0c52f51c2f2f1c4b9ef5322f51e3d4c6380640ad0bc7769893f0c network default NAD default] ADD starting CNI request [openshift-insights/insights-operator-5577457d66-z26lm 02e267c654c0c52f51c2f2f1c4b9ef5322f51e3d4c6380640ad0bc7769893f0c network default NAD default] I0329 02:51:45.648736 3885 cni.go:403] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-6fwbw 9ba2f62c6c1124048e6a6536984a986f21be8f578db85cae86f4b721bfb8a545 network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-6fwbw 9ba2f62c6c1124048e6a6536984a986f21be8f578db85cae86f4b721bfb8a545 network default NAD default] I0329 02:51:45.649143 3885 cni.go:403] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-9wxht 27984732f5e0cd32c34d4ea688c0ea90c7c5a564ff9067ce08cec4ada5e0e8e6 network default NAD default] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-9wxht 27984732f5e0cd32c34d4ea688c0ea90c7c5a564ff9067ce08cec4ada5e0e8e6 network default NAD default] I0329 02:51:45.653024 3885 cni.go:403] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t2dxv 605d4d075bbd81ea88cf075563fbb85244dd569759e13dcbb60bce2bdc51306d network default NAD default] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t2dxv 605d4d075bbd81ea88cf075563fbb85244dd569759e13dcbb60bce2bdc51306d network default NAD default] I0329 02:51:45.659507 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-8777bd9bb-9wdd5, hostIfaceName: 3a67aae0cb7daa9, network: default, NAD default, SandboxID: "3a67aae0cb7daa906b418c42e8c58ef90e8aa4a5afd20587fca5be5016e313df", PCI device ID: , UID: "63ae4464-29b9-4b28-810a-83a17558bcd4", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0329 02:51:45.669451 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-5577457d66-z26lm, hostIfaceName: 02e267c654c0c52, network: default, NAD default, SandboxID: "02e267c654c0c52f51c2f2f1c4b9ef5322f51e3d4c6380640ad0bc7769893f0c", PCI device ID: , UID: "540b459d-be15-4625-9fc5-2d23c5760032", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0329 02:51:45.670374 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-555cc59f77-9wxht, hostIfaceName: 27984732f5e0cd3, network: default, NAD default, SandboxID: "27984732f5e0cd32c34d4ea688c0ea90c7c5a564ff9067ce08cec4ada5e0e8e6", PCI device ID: , UID: "ef7a44d4-57c9-4d85-a7ed-a24c0237c139", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0329 02:51:45.670576 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-5c76bd6f9b-6fwbw, hostIfaceName: 9ba2f62c6c11240, network: default, NAD default, SandboxID: "9ba2f62c6c1124048e6a6536984a986f21be8f578db85cae86f4b721bfb8a545", PCI device ID: , UID: "bb95aaf7-e983-499f-a770-ec86213c072b", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0329 02:51:45.670962 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn, hostIfaceName: ffb6c07435bb4dc, network: default, NAD default, SandboxID: "ffb6c07435bb4dc139ed6b43b70615c60d90958ca9b36ea34436e28b50ee6ba6", PCI device ID: , UID: "98f91bb8-a901-4d1e-815b-f8104bb17ff8", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0329 02:51:45.671513 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-7b89bc695f-t2dxv, hostIfaceName: 605d4d075bbd81e, network: default, NAD default, SandboxID: "605d4d075bbd81ea88cf075563fbb85244dd569759e13dcbb60bce2bdc51306d", PCI device ID: , UID: "7493c811-2ee4-4940-889b-20efb1c3bad7", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0329 02:51:45.717206 3885 cni.go:424] [openshift-console-operator/console-operator-8777bd9bb-9wdd5 3a67aae0cb7daa906b418c42e8c58ef90e8aa4a5afd20587fca5be5016e313df network default NAD default] ADD finished CNI request [openshift-console-operator/console-operator-8777bd9bb-9wdd5 3a67aae0cb7daa906b418c42e8c58ef90e8aa4a5afd20587fca5be5016e313df network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:90:12:81:ae:12\",\"name\":\"3a67aae0cb7daa9\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/962a9e19-6a85-41da-abd3-b43b4b13c0b9\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:45.939023 3885 cni.go:424] [openshift-network-diagnostics/network-check-source-5c76bd6f9b-6fwbw 9ba2f62c6c1124048e6a6536984a986f21be8f578db85cae86f4b721bfb8a545 network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-5c76bd6f9b-6fwbw 9ba2f62c6c1124048e6a6536984a986f21be8f578db85cae86f4b721bfb8a545 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:e6:b2:eb:97:69\",\"name\":\"9ba2f62c6c11240\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6ec0653e-4962-4452-9ee6-5923acbef9b9\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:45.939601 3885 cni.go:424] [openshift-insights/insights-operator-5577457d66-z26lm 02e267c654c0c52f51c2f2f1c4b9ef5322f51e3d4c6380640ad0bc7769893f0c network default NAD default] ADD finished CNI request [openshift-insights/insights-operator-5577457d66-z26lm 02e267c654c0c52f51c2f2f1c4b9ef5322f51e3d4c6380640ad0bc7769893f0c network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:b3:a1:66:48:ab\",\"name\":\"02e267c654c0c52\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/80227ca4-9581-4353-a533-075c25bb219e\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:45.939840 3885 cni.go:424] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn ffb6c07435bb4dc139ed6b43b70615c60d90958ca9b36ea34436e28b50ee6ba6 network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-6ff64bb8b6-6sjcn ffb6c07435bb4dc139ed6b43b70615c60d90958ca9b36ea34436e28b50ee6ba6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"0a:d6:e2:5a:ac:ad\",\"name\":\"ffb6c07435bb4dc\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cefe7aa7-5069-426d-af8d-1db3ded82d8d\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:45.940300 3885 cni.go:424] [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t2dxv 605d4d075bbd81ea88cf075563fbb85244dd569759e13dcbb60bce2bdc51306d network default NAD default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-7b89bc695f-t2dxv 605d4d075bbd81ea88cf075563fbb85244dd569759e13dcbb60bce2bdc51306d network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:b3:b8:90:84:e9\",\"name\":\"605d4d075bbd81e\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/35e10810-0647-48c0-82fe-fa84407295f5\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:45.940722 3885 cni.go:424] [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-9wxht 27984732f5e0cd32c34d4ea688c0ea90c7c5a564ff9067ce08cec4ada5e0e8e6 network default NAD default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-555cc59f77-9wxht 27984732f5e0cd32c34d4ea688c0ea90c7c5a564ff9067ce08cec4ada5e0e8e6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:16:55:ff:1f:f6\",\"name\":\"27984732f5e0cd3\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/55ba019c-2113-4eb9-95c1-55859a43a0e2\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 02:51:50.703940 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0329 02:51:50.930640 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-37.ec2.internal in network "default" I0329 02:51:50.930691 3885 node_tracker.go:235] Node ip-10-0-1-37.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-37.ec2.internal" I0329 02:51:50.930710 3885 node_tracker.go:169] Node ip-10-0-1-37.ec2.internal switch + router changed, syncing services in network "default" I0329 02:51:50.930724 3885 services_controller.go:574] Full service sync requested for network=default I0329 02:51:50.930722 3885 default_network_controller.go:1008] Node "ip-10-0-1-37.ec2.internal" in remote zone "ip-10-0-1-37.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0329 02:51:50.931146 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0329 02:51:50.931161 3885 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0329 02:51:51.005783 3885 default_network_controller.go:1008] Node "ip-10-0-1-37.ec2.internal" in remote zone "ip-10-0-1-37.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:51:51.046022 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-37.ec2.internal in network "default" I0329 02:51:51.046086 3885 node_tracker.go:169] Node ip-10-0-1-37.ec2.internal switch + router changed, syncing services in network "default" I0329 02:51:51.046104 3885 services_controller.go:574] Full service sync requested for network=default I0329 02:51:51.046125 3885 default_network_controller.go:1008] Node "ip-10-0-1-37.ec2.internal" in remote zone "ip-10-0-1-37.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0329 02:51:51.046458 3885 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0329 02:51:51.046480 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0329 02:51:51.046803 3885 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-1-37.ec2.internal for the network default I0329 02:51:51.047921 3885 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-1-37.ec2.internal" on network "default" took: 1.13121ms I0329 02:51:51.396753 3885 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0329 02:51:51.397769 3885 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 995.6µs I0329 02:51:51.466418 3885 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0329 02:51:52.201855 3885 namespace.go:98] [openshift-service-ca] adding namespace I0329 02:51:52.202776 3885 namespace.go:102] [openshift-service-ca] adding namespace took 894.92µs I0329 02:51:52.231569 3885 namespace.go:142] [openshift-service-ca] updating namespace I0329 02:51:52.508762 3885 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-2gtbt] creating logical port openshift-kube-storage-version-migrator_migrator-854b5fb5c9-2gtbt for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:52.508874 3885 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-854b5fb5c9-2gtbt I0329 02:51:52.523666 3885 pod.go:62] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-2gtbt] pod update took 14.830406ms I0329 02:51:52.523687 3885 base_network_controller_pods.go:937] [default/openshift-kube-storage-version-migrator/migrator-854b5fb5c9-2gtbt] addLogicalPort annotation time took 14.855063ms I0329 02:51:52.524878 3885 pods.go:271] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-2gtbt] addLogicalPort took 16.134989ms, libovsdb time 821.552µs I0329 02:51:52.910222 3885 cni.go:403] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-2gtbt bbb0167a35b0ca96552b2d3c6cc915970c8f28c51927fcbd782e815ccb32d51e network default NAD default] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-2gtbt bbb0167a35b0ca96552b2d3c6cc915970c8f28c51927fcbd782e815ccb32d51e network default NAD default] I0329 02:51:52.918489 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-854b5fb5c9-2gtbt, hostIfaceName: bbb0167a35b0ca9, network: default, NAD default, SandboxID: "bbb0167a35b0ca96552b2d3c6cc915970c8f28c51927fcbd782e815ccb32d51e", PCI device ID: , UID: "dae91a09-361b-4322-ad1f-b107520ebe1c", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0329 02:51:52.959023 3885 cni.go:424] [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-2gtbt bbb0167a35b0ca96552b2d3c6cc915970c8f28c51927fcbd782e815ccb32d51e network default NAD default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-854b5fb5c9-2gtbt bbb0167a35b0ca96552b2d3c6cc915970c8f28c51927fcbd782e815ccb32d51e network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:72:d6:a3:1e:e6\",\"name\":\"bbb0167a35b0ca9\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/564f9b61-8881-41bf-9f30-2be55170966e\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:53.097060 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-37.ec2.internal in network "default" I0329 02:51:53.097076 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-37.ec2.internal I0329 02:51:53.097094 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-37.ec2.internal: 23.341µs I0329 02:51:53.097105 3885 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-37.ec2.internal", IP: 10.0.1.37 to PMTUD blocking rules I0329 02:51:53.097118 3885 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-37.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0329 02:51:53.097122 3885 node_tracker.go:169] Node ip-10-0-1-37.ec2.internal switch + router changed, syncing services in network "default" I0329 02:51:53.097135 3885 services_controller.go:574] Full service sync requested for network=default I0329 02:51:53.097062 3885 egressfirewall.go:628] Syncing node "ip-10-0-1-37.ec2.internal" for egress firewall W0329 02:51:53.097574 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0329 02:51:53.097574 3885 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0329 02:51:53.103373 3885 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-37.ec2.internal. Updating no re-route policies I0329 02:51:53.115216 3885 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-37.ec2.internal", IP: 10.0.1.37 to PMTUD blocking rules I0329 02:51:53.115232 3885 default_node_network_controller.go:1277] Adding remote node "ip-10-0-1-37.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0329 02:51:53.880425 3885 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-f59fc589b-pwqc8] creating logical port openshift-service-ca_service-ca-f59fc589b-pwqc8 for pod on switch ip-10-0-2-173.ec2.internal I0329 02:51:53.880622 3885 kube.go:256] Updating pod openshift-service-ca/service-ca-f59fc589b-pwqc8 I0329 02:51:53.894157 3885 pod.go:62] [openshift-service-ca/service-ca-f59fc589b-pwqc8] pod update took 13.572605ms I0329 02:51:53.894177 3885 base_network_controller_pods.go:937] [default/openshift-service-ca/service-ca-f59fc589b-pwqc8] addLogicalPort annotation time took 13.597452ms I0329 02:51:53.895531 3885 pods.go:271] [openshift-service-ca/service-ca-f59fc589b-pwqc8] addLogicalPort took 15.136879ms, libovsdb time 996.577µs I0329 02:51:54.405380 3885 cni.go:403] [openshift-service-ca/service-ca-f59fc589b-pwqc8 997c3239e3297672fd94b4ab5ffdc133d110e6a52d17b05a2cf9eb093fe03a38 network default NAD default] ADD starting CNI request [openshift-service-ca/service-ca-f59fc589b-pwqc8 997c3239e3297672fd94b4ab5ffdc133d110e6a52d17b05a2cf9eb093fe03a38 network default NAD default] I0329 02:51:54.413526 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-f59fc589b-pwqc8, hostIfaceName: 997c3239e329767, network: default, NAD default, SandboxID: "997c3239e3297672fd94b4ab5ffdc133d110e6a52d17b05a2cf9eb093fe03a38", PCI device ID: , UID: "34cf68ed-2db4-4165-9e47-2bed5278e5df", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0329 02:51:54.453904 3885 cni.go:424] [openshift-service-ca/service-ca-f59fc589b-pwqc8 997c3239e3297672fd94b4ab5ffdc133d110e6a52d17b05a2cf9eb093fe03a38 network default NAD default] ADD finished CNI request [openshift-service-ca/service-ca-f59fc589b-pwqc8 997c3239e3297672fd94b4ab5ffdc133d110e6a52d17b05a2cf9eb093fe03a38 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:a7:c9:3c:82:26\",\"name\":\"997c3239e329767\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f7011336-5490-4aab-9378-fe5d1f254617\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:51:56.595888 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-37.ec2.internal I0329 02:51:56.595927 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-37.ec2.internal: 45.379µs I0329 02:51:56.928451 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:51:56.928487 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 40.688µs I0329 02:51:56.928500 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-4.ec2.internal in network "default" I0329 02:51:56.928512 3885 node_tracker.go:220] Node ip-10-0-0-4.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0329 02:51:56.928530 3885 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall E0329 02:51:56.928560 3885 default_network_controller.go:816] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0329 02:51:56.928581 3885 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-4.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-4.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-4.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-4.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-4.ec2.internal, error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0329 02:51:56.955771 3885 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-4.ec2.internal annotation: unexpected end of JSON input I0329 02:51:56.955852 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0329 02:51:56.955888 3885 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0329 02:51:56.965818 3885 egressfirewall.go:628] Syncing node "ip-10-0-1-37.ec2.internal" for egress firewall I0329 02:51:56.965826 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-37.ec2.internal I0329 02:51:56.965918 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-37.ec2.internal: 94.907µs E0329 02:51:56.972541 3885 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-4.ec2.internal annotation: unexpected end of JSON input I0329 02:51:56.972577 3885 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall I0329 02:51:56.972604 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:51:56.972623 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 22.585µs I0329 02:51:56.972630 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0329 02:51:56.972662 3885 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0329 02:51:57.061263 3885 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-4.ec2.internal annotation: unexpected end of JSON input E0329 02:51:57.061329 3885 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-4.ec2.internal annotation: unexpected end of JSON input I0329 02:51:57.061362 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0329 02:51:57.061443 3885 default_network_controller.go:1025] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-4.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-4.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-4.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-4.ec2.internal" I0329 02:51:57.077732 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0329 02:51:57.190485 3885 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall I0329 02:51:57.190507 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:51:57.190520 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 17.228µs I0329 02:51:57.190583 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:51:57.248647 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:51:57.424095 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0329 02:51:59.581985 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0329 02:51:59.957318 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:52:00.161542 3885 cni.go:403] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-88thf 8027a2171d9eec1c38bb2035f203b1843a62ca21002216e9bc976b40d8b2c93a network default NAD default] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-88thf 8027a2171d9eec1c38bb2035f203b1843a62ca21002216e9bc976b40d8b2c93a network default NAD default] I0329 02:52:00.169882 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67b99c68bb-88thf, hostIfaceName: 8027a2171d9eec1, network: default, NAD default, SandboxID: "8027a2171d9eec1c38bb2035f203b1843a62ca21002216e9bc976b40d8b2c93a", PCI device ID: , UID: "c5a24064-43e8-4361-b011-22a4092dbe3c", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0329 02:52:00.211261 3885 cni.go:424] [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-88thf 8027a2171d9eec1c38bb2035f203b1843a62ca21002216e9bc976b40d8b2c93a network default NAD default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67b99c68bb-88thf 8027a2171d9eec1c38bb2035f203b1843a62ca21002216e9bc976b40d8b2c93a network default NAD default], result "{\"interfaces\":[{\"mac\":\"56:7f:b1:43:f2:bc\",\"name\":\"8027a2171d9eec1\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/47281b0e-18ab-4664-9047-5b0a02356fc9\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 02:52:00.364710 3885 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-7794f78cd7-7jq8s IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-7794f78cd7-7jq8s: no pod IPs found I0329 02:52:01.747393 3885 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c558b4c9b-8rqfz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:52:01.747436 3885 pods.go:176] Deleting pod: openshift-image-registry/image-registry-7c558b4c9b-8rqfz I0329 02:52:01.748921 3885 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-7c558b4c9b-8rqfz, ips: 10.128.0.6 I0329 02:52:01.748949 3885 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c558b4c9b-8rqfz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:52:01.757186 3885 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-64fc4f6495-5pq2g] creating logical port openshift-image-registry_image-registry-64fc4f6495-5pq2g for pod on switch ip-10-0-2-173.ec2.internal I0329 02:52:01.757304 3885 kube.go:256] Updating pod openshift-image-registry/image-registry-64fc4f6495-5pq2g I0329 02:52:01.765763 3885 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c558b4c9b-8rqfz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:52:01.765783 3885 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c558b4c9b-8rqfz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:52:01.773514 3885 pod.go:62] [openshift-image-registry/image-registry-64fc4f6495-5pq2g] pod update took 16.240059ms I0329 02:52:01.773532 3885 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-64fc4f6495-5pq2g] addLogicalPort annotation time took 16.263128ms I0329 02:52:01.774799 3885 pods.go:271] [openshift-image-registry/image-registry-64fc4f6495-5pq2g] addLogicalPort took 17.625607ms, libovsdb time 901.01µs I0329 02:52:01.775818 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c558b4c9b-8rqfz I0329 02:52:01.775837 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c558b4c9b-8rqfz I0329 02:52:02.170159 3885 cni.go:403] [openshift-image-registry/image-registry-64fc4f6495-5pq2g 2dd4d40e8527b30b50928f35a1674b66a126ae976702ee3a1d085be5c46bc8e2 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-64fc4f6495-5pq2g 2dd4d40e8527b30b50928f35a1674b66a126ae976702ee3a1d085be5c46bc8e2 network default NAD default] I0329 02:52:02.178335 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-64fc4f6495-5pq2g, hostIfaceName: 2dd4d40e8527b30, network: default, NAD default, SandboxID: "2dd4d40e8527b30b50928f35a1674b66a126ae976702ee3a1d085be5c46bc8e2", PCI device ID: , UID: "c8f8c8ab-30a8-4101-8ee9-12a5142e8afd", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0329 02:52:02.219491 3885 cni.go:424] [openshift-image-registry/image-registry-64fc4f6495-5pq2g 2dd4d40e8527b30b50928f35a1674b66a126ae976702ee3a1d085be5c46bc8e2 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-64fc4f6495-5pq2g 2dd4d40e8527b30b50928f35a1674b66a126ae976702ee3a1d085be5c46bc8e2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:97:7b:c8:5c:61\",\"name\":\"2dd4d40e8527b30\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6fe80cae-625d-433d-a3c2-f8e3e78b15a7\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 02:52:02.761506 3885 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:03.488151 3885 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:03.511698 3885 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0329 02:52:04.542590 3885 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-l2jnt] creating logical port openshift-insights_insights-runtime-extractor-l2jnt for pod on switch ip-10-0-2-173.ec2.internal I0329 02:52:04.542689 3885 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-l2jnt I0329 02:52:04.559385 3885 pod.go:62] [openshift-insights/insights-runtime-extractor-l2jnt] pod update took 16.730012ms I0329 02:52:04.559406 3885 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-l2jnt] addLogicalPort annotation time took 16.755074ms I0329 02:52:04.560604 3885 pods.go:271] [openshift-insights/insights-runtime-extractor-l2jnt] addLogicalPort took 18.030652ms, libovsdb time 836.755µs W0329 02:52:05.475964 3885 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:05.475986 3885 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:05.475993 3885 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:05.567318 3885 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:05.570967 3885 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-64fc4f6495-sbb2r IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-64fc4f6495-sbb2r: no pod IPs found I0329 02:52:05.624988 3885 cni.go:403] [openshift-dns/dns-default-wp5xk a25d2d5d54a0f1b4091e864dd6e088b816fc3bfb0440308aec94d16fea710f85 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-wp5xk a25d2d5d54a0f1b4091e864dd6e088b816fc3bfb0440308aec94d16fea710f85 network default NAD default] I0329 02:52:05.635889 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-wp5xk, hostIfaceName: a25d2d5d54a0f1b, network: default, NAD default, SandboxID: "a25d2d5d54a0f1b4091e864dd6e088b816fc3bfb0440308aec94d16fea710f85", PCI device ID: , UID: "a6a8291c-4b3d-4e89-95cf-5ef901a94942", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0329 02:52:05.677546 3885 cni.go:424] [openshift-dns/dns-default-wp5xk a25d2d5d54a0f1b4091e864dd6e088b816fc3bfb0440308aec94d16fea710f85 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-wp5xk a25d2d5d54a0f1b4091e864dd6e088b816fc3bfb0440308aec94d16fea710f85 network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:ea:ca:08:37:58\",\"name\":\"a25d2d5d54a0f1b\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/743276a5-f4ed-4361-a404-104cc0ce8210\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:06.457659 3885 cni.go:403] [openshift-insights/insights-runtime-extractor-l2jnt 0eede8b5bba6900add186d45c8cd2e591cccc412bf615f669a05a332015eb77d network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-l2jnt 0eede8b5bba6900add186d45c8cd2e591cccc412bf615f669a05a332015eb77d network default NAD default] I0329 02:52:06.466165 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-l2jnt, hostIfaceName: 0eede8b5bba6900, network: default, NAD default, SandboxID: "0eede8b5bba6900add186d45c8cd2e591cccc412bf615f669a05a332015eb77d", PCI device ID: , UID: "6fa36ada-476a-4648-8e24-171885d7a469", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0329 02:52:06.509837 3885 cni.go:424] [openshift-insights/insights-runtime-extractor-l2jnt 0eede8b5bba6900add186d45c8cd2e591cccc412bf615f669a05a332015eb77d network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-l2jnt 0eede8b5bba6900add186d45c8cd2e591cccc412bf615f669a05a332015eb77d network default NAD default], result "{\"interfaces\":[{\"mac\":\"4e:df:0b:c7:23:c1\",\"name\":\"0eede8b5bba6900\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1934e5d6-84e3-4380-9e21-0107eb1c36ef\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:07.042979 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:52:07.723086 3885 cni.go:403] [openshift-multus/network-metrics-daemon-2jw6j b1efcec80900fdc881744a8abbe5c0319c58f52e44744efadcff0e87ad812c18 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-2jw6j b1efcec80900fdc881744a8abbe5c0319c58f52e44744efadcff0e87ad812c18 network default NAD default] I0329 02:52:07.734789 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-2jw6j, hostIfaceName: b1efcec80900fdc, network: default, NAD default, SandboxID: "b1efcec80900fdc881744a8abbe5c0319c58f52e44744efadcff0e87ad812c18", PCI device ID: , UID: "e238b82e-64b4-4cf3-b5db-c45860119045", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0329 02:52:07.788552 3885 cni.go:424] [openshift-multus/network-metrics-daemon-2jw6j b1efcec80900fdc881744a8abbe5c0319c58f52e44744efadcff0e87ad812c18 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-2jw6j b1efcec80900fdc881744a8abbe5c0319c58f52e44744efadcff0e87ad812c18 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:c7:f0:b4:b5:b8\",\"name\":\"b1efcec80900fdc\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7ff6997f-a25a-4f1c-bb24-7a44fb66c176\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:08.626518 3885 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-9bdvq] creating logical port openshift-console_downloads-56d45c498b-9bdvq for pod on switch ip-10-0-2-173.ec2.internal I0329 02:52:08.626630 3885 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-9bdvq I0329 02:52:08.642854 3885 pod.go:62] [openshift-console/downloads-56d45c498b-9bdvq] pod update took 16.244349ms I0329 02:52:08.642945 3885 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-9bdvq] addLogicalPort annotation time took 16.338222ms I0329 02:52:08.644362 3885 pods.go:271] [openshift-console/downloads-56d45c498b-9bdvq] addLogicalPort took 17.8676ms, libovsdb time 959.844µs W0329 02:52:08.827116 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0329 02:52:08.855188 3885 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:08.855207 3885 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:08.855213 3885 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0329 02:52:09.094534 3885 cni.go:403] [openshift-console/downloads-56d45c498b-9bdvq 8aac83cc87dbe66ae66d1bd4f0114b28a8b03b9b52543421a5e11222d37e0f82 network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-9bdvq 8aac83cc87dbe66ae66d1bd4f0114b28a8b03b9b52543421a5e11222d37e0f82 network default NAD default] I0329 02:52:09.102777 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-9bdvq, hostIfaceName: 8aac83cc87dbe66, network: default, NAD default, SandboxID: "8aac83cc87dbe66ae66d1bd4f0114b28a8b03b9b52543421a5e11222d37e0f82", PCI device ID: , UID: "4633eb1e-a33e-4e77-a4f1-4674ae7a22a6", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0329 02:52:09.144875 3885 cni.go:424] [openshift-console/downloads-56d45c498b-9bdvq 8aac83cc87dbe66ae66d1bd4f0114b28a8b03b9b52543421a5e11222d37e0f82 network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-9bdvq 8aac83cc87dbe66ae66d1bd4f0114b28a8b03b9b52543421a5e11222d37e0f82 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:49:02:d6:79:fc\",\"name\":\"8aac83cc87dbe66\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/38efce94-4934-4f96-8346-ba1e9c4d5baa\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:16.306505 3885 cni.go:403] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-n6krz bd70329c9d0b7fd0cd019cc7a74a756a00b102a7549d3d2792fbc5c99fc05107 network default NAD default] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-n6krz bd70329c9d0b7fd0cd019cc7a74a756a00b102a7549d3d2792fbc5c99fc05107 network default NAD default] I0329 02:52:16.316434 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-6dd897695f-n6krz, hostIfaceName: bd70329c9d0b7fd, network: default, NAD default, SandboxID: "bd70329c9d0b7fd0cd019cc7a74a756a00b102a7549d3d2792fbc5c99fc05107", PCI device ID: , UID: "f0c69e90-f1be-4d82-97fb-d28315a3bf20", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0329 02:52:16.357878 3885 cni.go:403] [openshift-ingress/router-default-84747c8b65-sn66l b5072c6cdf8c031296a5ed3ec77a542bfe164b8794e6e60bfca448345d95b710 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-84747c8b65-sn66l b5072c6cdf8c031296a5ed3ec77a542bfe164b8794e6e60bfca448345d95b710 network default NAD default] I0329 02:52:16.361953 3885 cni.go:403] [openshift-ingress/router-default-7bf7699dd9-knnqb bf31ccc527a714654df40fde17840d1e5885d6dde389cfbc7d90c1406d438c1c network default NAD default] ADD starting CNI request [openshift-ingress/router-default-7bf7699dd9-knnqb bf31ccc527a714654df40fde17840d1e5885d6dde389cfbc7d90c1406d438c1c network default NAD default] I0329 02:52:16.367064 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-84747c8b65-sn66l, hostIfaceName: b5072c6cdf8c031, network: default, NAD default, SandboxID: "b5072c6cdf8c031296a5ed3ec77a542bfe164b8794e6e60bfca448345d95b710", PCI device ID: , UID: "cdc4bf71-f7a3-40a9-867a-1be2350d53c5", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0329 02:52:16.368340 3885 cni.go:424] [openshift-monitoring/cluster-monitoring-operator-6dd897695f-n6krz bd70329c9d0b7fd0cd019cc7a74a756a00b102a7549d3d2792fbc5c99fc05107 network default NAD default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-6dd897695f-n6krz bd70329c9d0b7fd0cd019cc7a74a756a00b102a7549d3d2792fbc5c99fc05107 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:0a:c4:36:57:dc\",\"name\":\"bd70329c9d0b7fd\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f7620d38-d253-42a6-ab01-be4563117bfc\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:16.371806 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7bf7699dd9-knnqb, hostIfaceName: bf31ccc527a7146, network: default, NAD default, SandboxID: "bf31ccc527a714654df40fde17840d1e5885d6dde389cfbc7d90c1406d438c1c", PCI device ID: , UID: "d522b87a-e4b9-44a6-98e4-fd866967de05", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0329 02:52:16.419175 3885 cni.go:424] [openshift-ingress/router-default-84747c8b65-sn66l b5072c6cdf8c031296a5ed3ec77a542bfe164b8794e6e60bfca448345d95b710 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-84747c8b65-sn66l b5072c6cdf8c031296a5ed3ec77a542bfe164b8794e6e60bfca448345d95b710 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:44:7b:24:bc:2e\",\"name\":\"b5072c6cdf8c031\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3568ffe7-5ff5-425e-99f5-913312f6eff5\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:16.436770 3885 cni.go:424] [openshift-ingress/router-default-7bf7699dd9-knnqb bf31ccc527a714654df40fde17840d1e5885d6dde389cfbc7d90c1406d438c1c network default NAD default] ADD finished CNI request [openshift-ingress/router-default-7bf7699dd9-knnqb bf31ccc527a714654df40fde17840d1e5885d6dde389cfbc7d90c1406d438c1c network default NAD default], result "{\"interfaces\":[{\"mac\":\"fe:55:f2:86:63:90\",\"name\":\"bf31ccc527a7146\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/10189a07-b7fe-4040-8f0d-a6450426e469\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:16.515541 3885 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-x5tgn 29539c71778201f901905e5c34e48a4614dbba28411d28d0e7c4aea63b86b7a1 network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-x5tgn 29539c71778201f901905e5c34e48a4614dbba28411d28d0e7c4aea63b86b7a1 network default NAD default] I0329 02:52:16.521011 3885 cni.go:403] [openshift-network-console/networking-console-plugin-58d8f6745c-7zxcs baf176200f159ed9e35129006d86bd2e0bf505892394d824e80dcdb062b9074c network default NAD default] ADD starting CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-7zxcs baf176200f159ed9e35129006d86bd2e0bf505892394d824e80dcdb062b9074c network default NAD default] I0329 02:52:16.526593 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-x5tgn, hostIfaceName: 29539c71778201f, network: default, NAD default, SandboxID: "29539c71778201f901905e5c34e48a4614dbba28411d28d0e7c4aea63b86b7a1", PCI device ID: , UID: "68c6f96c-bdf4-4e66-839c-593ad73e45a0", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0329 02:52:16.538327 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-58d8f6745c-7zxcs, hostIfaceName: baf176200f159ed, network: default, NAD default, SandboxID: "baf176200f159ed9e35129006d86bd2e0bf505892394d824e80dcdb062b9074c", PCI device ID: , UID: "02015556-7245-4a3d-a384-5d9d19b32f6d", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0329 02:52:16.582691 3885 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-x5tgn 29539c71778201f901905e5c34e48a4614dbba28411d28d0e7c4aea63b86b7a1 network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-x5tgn 29539c71778201f901905e5c34e48a4614dbba28411d28d0e7c4aea63b86b7a1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"be:83:4f:35:77:ab\",\"name\":\"29539c71778201f\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fd6d176a-829d-4d79-8794-78f1618c358f\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:16.603588 3885 cni.go:424] [openshift-network-console/networking-console-plugin-58d8f6745c-7zxcs baf176200f159ed9e35129006d86bd2e0bf505892394d824e80dcdb062b9074c network default NAD default] ADD finished CNI request [openshift-network-console/networking-console-plugin-58d8f6745c-7zxcs baf176200f159ed9e35129006d86bd2e0bf505892394d824e80dcdb062b9074c network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:aa:11:63:48:2d\",\"name\":\"baf176200f159ed\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7e64e46e-a948-4b59-a43e-7d8ce35e1420\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:16.856058 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:52:16.856073 3885 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall I0329 02:52:16.856081 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 32.334µs I0329 02:52:16.856143 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:52:28.021790 3885 obj_retry.go:300] Retry object setup: *v1.Node ip-10-0-0-4.ec2.internal I0329 02:52:28.021828 3885 obj_retry.go:364] Adding new object: *v1.Node ip-10-0-0-4.ec2.internal I0329 02:52:28.021927 3885 obj_retry.go:375] Retry add failed for *v1.Node ip-10-0-0-4.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-4.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-4.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-4.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-4.ec2.internal I0329 02:52:29.545007 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-xxzzq I0329 02:52:29.545039 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-xxzzq I0329 02:52:29.545084 3885 obj_retry.go:375] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-xxzzq, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-xxzzq: suppressed error logged: pod openshift-network-diagnostics/network-check-target-xxzzq: no pod IPs found I0329 02:52:29.545007 3885 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-wwhgw I0329 02:52:29.545097 3885 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-wwhgw I0329 02:52:29.545115 3885 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-wwhgw, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-wwhgw: suppressed error logged: pod openshift-multus/network-metrics-daemon-wwhgw: no pod IPs found I0329 02:52:30.035541 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0329 02:52:30.039781 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0329 02:52:30.056883 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:30.056903 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0329 02:52:30.056908 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0329 02:52:30.101025 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0329 02:52:30.203393 3885 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0329 02:52:30.212715 3885 base_network_controller_pods.go:487] [default/openshift-console/console-7677b9b887-84h62] creating logical port openshift-console_console-7677b9b887-84h62 for pod on switch ip-10-0-2-173.ec2.internal I0329 02:52:30.212872 3885 kube.go:256] Updating pod openshift-console/console-7677b9b887-84h62 I0329 02:52:30.233338 3885 pod.go:62] [openshift-console/console-7677b9b887-84h62] pod update took 20.501272ms I0329 02:52:30.233363 3885 base_network_controller_pods.go:937] [default/openshift-console/console-7677b9b887-84h62] addLogicalPort annotation time took 20.532009ms I0329 02:52:30.234560 3885 pods.go:271] [openshift-console/console-7677b9b887-84h62] addLogicalPort took 21.864105ms, libovsdb time 820.415µs I0329 02:52:30.240312 3885 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xtprn] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-xtprn for pod on switch ip-10-0-2-173.ec2.internal I0329 02:52:30.240406 3885 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xtprn I0329 02:52:30.264717 3885 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xtprn] pod update took 24.340322ms I0329 02:52:30.264740 3885 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xtprn] addLogicalPort annotation time took 24.366633ms I0329 02:52:30.265985 3885 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xtprn] addLogicalPort took 25.687657ms, libovsdb time 896.122µs I0329 02:52:30.583505 3885 cni.go:403] [openshift-console/console-7677b9b887-84h62 5594c4ec02eb587ac53c1aa1fbc7f2ff03425626fdffcffd630a7e9392a4767f network default NAD default] ADD starting CNI request [openshift-console/console-7677b9b887-84h62 5594c4ec02eb587ac53c1aa1fbc7f2ff03425626fdffcffd630a7e9392a4767f network default NAD default] I0329 02:52:30.592905 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-7677b9b887-84h62, hostIfaceName: 5594c4ec02eb587, network: default, NAD default, SandboxID: "5594c4ec02eb587ac53c1aa1fbc7f2ff03425626fdffcffd630a7e9392a4767f", PCI device ID: , UID: "4333461f-9b39-4bd1-b8cd-4f493b0304ae", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0329 02:52:30.608750 3885 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xtprn a7e9fdd933babc3c34bd508396fd0a77e8cba5307daa0362e4ce4d72f286a87c network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xtprn a7e9fdd933babc3c34bd508396fd0a77e8cba5307daa0362e4ce4d72f286a87c network default NAD default] I0329 02:52:30.619246 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-xtprn, hostIfaceName: a7e9fdd933babc3, network: default, NAD default, SandboxID: "a7e9fdd933babc3c34bd508396fd0a77e8cba5307daa0362e4ce4d72f286a87c", PCI device ID: , UID: "fb39a5d2-faf8-4054-bd52-66548ed349fa", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0329 02:52:30.645490 3885 cni.go:424] [openshift-console/console-7677b9b887-84h62 5594c4ec02eb587ac53c1aa1fbc7f2ff03425626fdffcffd630a7e9392a4767f network default NAD default] ADD finished CNI request [openshift-console/console-7677b9b887-84h62 5594c4ec02eb587ac53c1aa1fbc7f2ff03425626fdffcffd630a7e9392a4767f network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:c4:5e:6f:96:f6\",\"name\":\"5594c4ec02eb587\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/644ceecc-183f-4d5c-a307-cf95a377b5eb\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:30.666113 3885 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xtprn a7e9fdd933babc3c34bd508396fd0a77e8cba5307daa0362e4ce4d72f286a87c network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-xtprn a7e9fdd933babc3c34bd508396fd0a77e8cba5307daa0362e4ce4d72f286a87c network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:c1:d5:bc:b4:ef\",\"name\":\"a7e9fdd933babc3\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6c1f103f-4cd8-4e74-978b-bf40ba76b498\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 02:52:30.883581 3885 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c558b4c9b-jdfv5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:52:30.884152 3885 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7c558b4c9b-jdfv5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0329 02:52:30.895126 3885 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:31.051348 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-4.ec2.internal in network "default" I0329 02:52:31.051394 3885 node_tracker.go:235] Node ip-10-0-0-4.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-4.ec2.internal" I0329 02:52:31.051412 3885 node_tracker.go:169] Node ip-10-0-0-4.ec2.internal switch + router changed, syncing services in network "default" I0329 02:52:31.051423 3885 services_controller.go:574] Full service sync requested for network=default I0329 02:52:31.051429 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "ip-10-0-0-4.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0329 02:52:31.051939 3885 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:31.205154 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-4.ec2.internal in network "default" I0329 02:52:31.205219 3885 node_tracker.go:169] Node ip-10-0-0-4.ec2.internal switch + router changed, syncing services in network "default" I0329 02:52:31.205234 3885 services_controller.go:574] Full service sync requested for network=default I0329 02:52:31.205151 3885 default_network_controller.go:1008] Node "ip-10-0-0-4.ec2.internal" in remote zone "ip-10-0-0-4.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0329 02:52:31.205802 3885 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:31.206013 3885 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-4.ec2.internal for the network default I0329 02:52:31.207269 3885 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-4.ec2.internal" on network "default" took: 1.270015ms I0329 02:52:31.558885 3885 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c558b4c9b-jdfv5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:52:31.558911 3885 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7c558b4c9b-jdfv5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:52:31.569880 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7c558b4c9b-jdfv5 I0329 02:52:31.569943 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7c558b4c9b-jdfv5 I0329 02:52:31.756764 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:52:31.756791 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 34.706µs I0329 02:52:31.756769 3885 egressfirewall.go:628] Syncing node "ip-10-0-0-4.ec2.internal" for egress firewall I0329 02:52:31.756829 3885 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-4.ec2.internal in network "default" I0329 02:52:31.756881 3885 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-4.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0329 02:52:31.756892 3885 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-4.ec2.internal", IP: 10.0.0.4 to PMTUD blocking rules I0329 02:52:31.756892 3885 node_tracker.go:169] Node ip-10-0-0-4.ec2.internal switch + router changed, syncing services in network "default" I0329 02:52:31.756910 3885 services_controller.go:574] Full service sync requested for network=default W0329 02:52:31.757346 3885 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0329 02:52:31.764264 3885 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-4.ec2.internal. Updating no re-route policies I0329 02:52:31.774683 3885 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-4.ec2.internal", IP: 10.0.0.4 to PMTUD blocking rules I0329 02:52:31.774699 3885 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-4.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0329 02:52:32.584756 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0329 02:52:32.615935 3885 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:32.952123 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0329 02:52:33.783854 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0329 02:52:33.936835 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0329 02:52:34.598283 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0329 02:52:36.258914 3885 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-4.ec2.internal I0329 02:52:36.258941 3885 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-4.ec2.internal: 34.317µs W0329 02:52:40.625651 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0329 02:52:40.625675 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0329 02:52:40.971878 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0329 02:52:43.648101 3885 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:52:43.648126 3885 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:52:43.648133 3885 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0329 02:52:44.178294 3885 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0329 02:52:45.109304 3885 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:52:45.109323 3885 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:52:45.109328 3885 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0329 02:52:47.671237 3885 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:47.671262 3885 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:47.671269 3885 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:47.671277 3885 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0329 02:52:47.694906 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0329 02:52:47.788064 3885 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:52:47.788080 3885 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:52:51.157364 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0329 02:52:51.228760 3885 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:52:51.228775 3885 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0329 02:52:51.728972 3885 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-d477cd6dd-f9sdw IP address from the namespace address-set, err: pod openshift-console/console-d477cd6dd-f9sdw: no pod IPs found W0329 02:53:01.052606 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0329 02:53:01.052626 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0329 02:53:01.052658 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0329 02:53:02.077227 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0329 02:53:09.525409 3885 obj_retry.go:464] Detected object openshift-console/console-7677b9b887-frl82 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:09.525957 3885 obj_retry.go:464] Detected object openshift-console/console-7677b9b887-frl82 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:09.763286 3885 obj_retry.go:458] Detected object openshift-console/console-7677b9b887-frl82 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:09.763312 3885 obj_retry.go:458] Detected object openshift-console/console-7677b9b887-frl82 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:09.771259 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7677b9b887-frl82 I0329 02:53:09.771279 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7677b9b887-frl82 I0329 02:53:12.079659 3885 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf for pod on switch ip-10-0-2-173.ec2.internal I0329 02:53:12.079767 3885 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf I0329 02:53:12.092990 3885 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf] pod update took 13.250877ms I0329 02:53:12.093009 3885 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf] addLogicalPort annotation time took 13.278034ms I0329 02:53:12.094199 3885 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf] addLogicalPort took 14.557917ms, libovsdb time 854.21µs W0329 02:53:12.111606 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0329 02:53:12.111634 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0329 02:53:12.111655 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0329 02:53:12.489291 3885 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf 74ecf4fa04478894ed4bb34b42f8007b54a17c8ac75da299363092dbbb97fb98 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf 74ecf4fa04478894ed4bb34b42f8007b54a17c8ac75da299363092dbbb97fb98 network default NAD default] I0329 02:53:12.497933 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf, hostIfaceName: 74ecf4fa0447889, network: default, NAD default, SandboxID: "74ecf4fa04478894ed4bb34b42f8007b54a17c8ac75da299363092dbbb97fb98", PCI device ID: , UID: "a4c38a98-a5b7-49f2-bc5e-8929ec204a0b", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0329 02:53:12.542733 3885 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf 74ecf4fa04478894ed4bb34b42f8007b54a17c8ac75da299363092dbbb97fb98 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf 74ecf4fa04478894ed4bb34b42f8007b54a17c8ac75da299363092dbbb97fb98 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5e:d3:e7:ae:48:6e\",\"name\":\"74ecf4fa0447889\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/35d1847f-18ce-44c0-bc47-0d86da58f489\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 02:53:13.106935 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0329 02:53:18.140961 3885 cni.go:403] [openshift-image-registry/image-registry-64fc4f6495-5pq2g 2dd4d40e8527b30b50928f35a1674b66a126ae976702ee3a1d085be5c46bc8e2 network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-64fc4f6495-5pq2g 2dd4d40e8527b30b50928f35a1674b66a126ae976702ee3a1d085be5c46bc8e2 network default NAD default] I0329 02:53:18.212112 3885 cni.go:424] [openshift-image-registry/image-registry-64fc4f6495-5pq2g 2dd4d40e8527b30b50928f35a1674b66a126ae976702ee3a1d085be5c46bc8e2 network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-64fc4f6495-5pq2g 2dd4d40e8527b30b50928f35a1674b66a126ae976702ee3a1d085be5c46bc8e2 network default NAD default], result "{}", err I0329 02:53:18.281543 3885 obj_retry.go:464] Detected object openshift-image-registry/image-registry-64fc4f6495-5pq2g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:18.281581 3885 pods.go:176] Deleting pod: openshift-image-registry/image-registry-64fc4f6495-5pq2g I0329 02:53:18.282854 3885 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-64fc4f6495-5pq2g, ips: 10.128.0.25 I0329 02:53:18.282883 3885 obj_retry.go:464] Detected object openshift-image-registry/image-registry-64fc4f6495-5pq2g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:18.435130 3885 obj_retry.go:464] Detected object openshift-console/console-d477cd6dd-plpzq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:18.435603 3885 obj_retry.go:464] Detected object openshift-console/console-d477cd6dd-plpzq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:19.148452 3885 obj_retry.go:458] Detected object openshift-image-registry/image-registry-64fc4f6495-5pq2g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:19.148498 3885 obj_retry.go:458] Detected object openshift-image-registry/image-registry-64fc4f6495-5pq2g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:19.156438 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-64fc4f6495-5pq2g I0329 02:53:19.156483 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-64fc4f6495-5pq2g I0329 02:53:19.262958 3885 obj_retry.go:458] Detected object openshift-console/console-d477cd6dd-plpzq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:19.262985 3885 obj_retry.go:458] Detected object openshift-console/console-d477cd6dd-plpzq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:19.271882 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-d477cd6dd-plpzq I0329 02:53:19.271973 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-d477cd6dd-plpzq W0329 02:53:21.164710 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0329 02:53:21.164730 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0329 02:53:21.164772 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0329 02:53:22.164281 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0329 02:53:27.193951 3885 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf 74ecf4fa04478894ed4bb34b42f8007b54a17c8ac75da299363092dbbb97fb98 network default NAD default] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf 74ecf4fa04478894ed4bb34b42f8007b54a17c8ac75da299363092dbbb97fb98 network default NAD default] I0329 02:53:27.270220 3885 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf 74ecf4fa04478894ed4bb34b42f8007b54a17c8ac75da299363092dbbb97fb98 network default NAD default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf 74ecf4fa04478894ed4bb34b42f8007b54a17c8ac75da299363092dbbb97fb98 network default NAD default], result "{}", err I0329 02:53:27.317095 3885 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:27.317129 3885 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf I0329 02:53:27.318552 3885 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf, ips: 10.128.0.30 I0329 02:53:27.318585 3885 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:27.318607 3885 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 02:53:27.318619 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0329 02:53:27.318807 3885 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 02:53:27.318818 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0329 02:53:28.345833 3885 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:28.345865 3885 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:28.345884 3885 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:28.345902 3885 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf565390s2hkf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:30.204022 3885 cni.go:403] [openshift-ingress/router-default-7bf7699dd9-knnqb bf31ccc527a714654df40fde17840d1e5885d6dde389cfbc7d90c1406d438c1c network default NAD default] DEL starting CNI request [openshift-ingress/router-default-7bf7699dd9-knnqb bf31ccc527a714654df40fde17840d1e5885d6dde389cfbc7d90c1406d438c1c network default NAD default] I0329 02:53:30.275546 3885 cni.go:424] [openshift-ingress/router-default-7bf7699dd9-knnqb bf31ccc527a714654df40fde17840d1e5885d6dde389cfbc7d90c1406d438c1c network default NAD default] DEL finished CNI request [openshift-ingress/router-default-7bf7699dd9-knnqb bf31ccc527a714654df40fde17840d1e5885d6dde389cfbc7d90c1406d438c1c network default NAD default], result "{}", err I0329 02:53:30.325301 3885 obj_retry.go:464] Detected object openshift-ingress/router-default-7bf7699dd9-knnqb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:30.325335 3885 pods.go:176] Deleting pod: openshift-ingress/router-default-7bf7699dd9-knnqb I0329 02:53:30.326799 3885 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-7bf7699dd9-knnqb, ips: 10.128.0.16 I0329 02:53:30.326829 3885 obj_retry.go:464] Detected object openshift-ingress/router-default-7bf7699dd9-knnqb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:31.212781 3885 obj_retry.go:458] Detected object openshift-ingress/router-default-7bf7699dd9-knnqb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:31.212812 3885 obj_retry.go:458] Detected object openshift-ingress/router-default-7bf7699dd9-knnqb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:31.224236 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-7bf7699dd9-knnqb I0329 02:53:31.224266 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-7bf7699dd9-knnqb I0329 02:53:32.153183 3885 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0329 02:53:32.163287 3885 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0329 02:53:46.051072 3885 cni.go:403] [openshift-console/console-7677b9b887-84h62 5594c4ec02eb587ac53c1aa1fbc7f2ff03425626fdffcffd630a7e9392a4767f network default NAD default] DEL starting CNI request [openshift-console/console-7677b9b887-84h62 5594c4ec02eb587ac53c1aa1fbc7f2ff03425626fdffcffd630a7e9392a4767f network default NAD default] I0329 02:53:46.130964 3885 cni.go:424] [openshift-console/console-7677b9b887-84h62 5594c4ec02eb587ac53c1aa1fbc7f2ff03425626fdffcffd630a7e9392a4767f network default NAD default] DEL finished CNI request [openshift-console/console-7677b9b887-84h62 5594c4ec02eb587ac53c1aa1fbc7f2ff03425626fdffcffd630a7e9392a4767f network default NAD default], result "{}", err I0329 02:53:46.177082 3885 obj_retry.go:464] Detected object openshift-console/console-7677b9b887-84h62 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:46.177112 3885 pods.go:176] Deleting pod: openshift-console/console-7677b9b887-84h62 I0329 02:53:46.178722 3885 pods.go:236] Attempting to release IPs for pod: openshift-console/console-7677b9b887-84h62, ips: 10.128.0.28 I0329 02:53:46.178759 3885 obj_retry.go:464] Detected object openshift-console/console-7677b9b887-84h62 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:53:46.523649 3885 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0329 02:53:46.578011 3885 obj_retry.go:458] Detected object openshift-console/console-7677b9b887-84h62 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:46.578035 3885 obj_retry.go:458] Detected object openshift-console/console-7677b9b887-84h62 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:53:46.586163 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7677b9b887-84h62 I0329 02:53:46.586192 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7677b9b887-84h62 I0329 02:54:08.759727 3885 base_network_controller_pods.go:487] [default/openshift-console/console-f66787c65-pdzzb] creating logical port openshift-console_console-f66787c65-pdzzb for pod on switch ip-10-0-2-173.ec2.internal I0329 02:54:08.759856 3885 kube.go:256] Updating pod openshift-console/console-f66787c65-pdzzb I0329 02:54:08.785891 3885 pod.go:62] [openshift-console/console-f66787c65-pdzzb] pod update took 26.06205ms I0329 02:54:08.785917 3885 base_network_controller_pods.go:937] [default/openshift-console/console-f66787c65-pdzzb] addLogicalPort annotation time took 26.093429ms I0329 02:54:08.787097 3885 pods.go:271] [openshift-console/console-f66787c65-pdzzb] addLogicalPort took 27.387164ms, libovsdb time 786.482µs I0329 02:54:09.122393 3885 cni.go:403] [openshift-console/console-f66787c65-pdzzb a5372e788a70017017d43162f1bb4ac38399bcc90ae5f700ed01a91f9e8d9bb8 network default NAD default] ADD starting CNI request [openshift-console/console-f66787c65-pdzzb a5372e788a70017017d43162f1bb4ac38399bcc90ae5f700ed01a91f9e8d9bb8 network default NAD default] I0329 02:54:09.130730 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-f66787c65-pdzzb, hostIfaceName: a5372e788a70017, network: default, NAD default, SandboxID: "a5372e788a70017017d43162f1bb4ac38399bcc90ae5f700ed01a91f9e8d9bb8", PCI device ID: , UID: "63c3c14e-bdfd-4ab9-86a0-ccbd144ddfd8", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0329 02:54:09.174965 3885 cni.go:424] [openshift-console/console-f66787c65-pdzzb a5372e788a70017017d43162f1bb4ac38399bcc90ae5f700ed01a91f9e8d9bb8 network default NAD default] ADD finished CNI request [openshift-console/console-f66787c65-pdzzb a5372e788a70017017d43162f1bb4ac38399bcc90ae5f700ed01a91f9e8d9bb8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b6:bd:e4:82:2a:95\",\"name\":\"a5372e788a70017\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a5e4ed49-6997-4f5f-9a43-cf0222b1c899\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 02:54:19.447745 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0329 02:54:34.071951 3885 obj_retry.go:464] Detected object openshift-console/console-86449784f9-82tnj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:34.072569 3885 obj_retry.go:464] Detected object openshift-console/console-86449784f9-82tnj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:34.391406 3885 obj_retry.go:458] Detected object openshift-console/console-86449784f9-82tnj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:34.391429 3885 obj_retry.go:458] Detected object openshift-console/console-86449784f9-82tnj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:34.397926 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86449784f9-82tnj I0329 02:54:34.397950 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86449784f9-82tnj I0329 02:54:40.543881 3885 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 3213a5c2796137c4633b43fdf23ae105a4a0f3028664ed2d03d28d699f1077ac network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 3213a5c2796137c4633b43fdf23ae105a4a0f3028664ed2d03d28d699f1077ac network default NAD default] I0329 02:54:40.625386 3885 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 3213a5c2796137c4633b43fdf23ae105a4a0f3028664ed2d03d28d699f1077ac network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 3213a5c2796137c4633b43fdf23ae105a4a0f3028664ed2d03d28d699f1077ac network default NAD default], result "{}", err I0329 02:54:40.691675 3885 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:40.691703 3885 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 I0329 02:54:40.693110 3885 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7, ips: 10.128.0.9 I0329 02:54:40.693136 3885 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:41.716778 3885 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:41.716802 3885 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579210-c26s7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:44.765738 3885 obj_retry.go:464] Detected object openshift-console/console-86449784f9-k4vn9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:44.766320 3885 obj_retry.go:464] Detected object openshift-console/console-86449784f9-k4vn9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 02:54:45.568566 3885 obj_retry.go:458] Detected object openshift-console/console-86449784f9-k4vn9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:45.568590 3885 obj_retry.go:458] Detected object openshift-console/console-86449784f9-k4vn9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 02:54:45.575791 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86449784f9-k4vn9 I0329 02:54:45.575809 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86449784f9-k4vn9 I0329 02:56:41.559492 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=13 I0329 02:56:49.671345 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0329 02:57:06.665445 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0329 02:57:08.555065 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=110 I0329 02:57:08.555076 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=70 I0329 02:57:42.644252 3885 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0329 02:58:23.551862 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=79 I0329 02:58:33.229803 3885 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-5q8cz] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-5q8cz for pod on switch ip-10-0-2-173.ec2.internal I0329 02:58:33.229935 3885 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-5q8cz I0329 02:58:33.258399 3885 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-5q8cz] pod update took 28.493219ms I0329 02:58:33.258431 3885 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-5q8cz] addLogicalPort annotation time took 28.52966ms I0329 02:58:33.259766 3885 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-5q8cz] addLogicalPort took 29.981938ms, libovsdb time 953.876µs I0329 02:58:33.587247 3885 cni.go:403] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-5q8cz 8758e3eba29b21f929e8f5b6d55182bd205ccd51843cda4fe985dc830016dc70 network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-5q8cz 8758e3eba29b21f929e8f5b6d55182bd205ccd51843cda4fe985dc830016dc70 network default NAD default] I0329 02:58:33.595835 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-5q8cz, hostIfaceName: 8758e3eba29b21f, network: default, NAD default, SandboxID: "8758e3eba29b21f929e8f5b6d55182bd205ccd51843cda4fe985dc830016dc70", PCI device ID: , UID: "72959a83-b9a6-4d62-9693-373601c03db8", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0329 02:58:33.639770 3885 cni.go:424] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-5q8cz 8758e3eba29b21f929e8f5b6d55182bd205ccd51843cda4fe985dc830016dc70 network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-5q8cz 8758e3eba29b21f929e8f5b6d55182bd205ccd51843cda4fe985dc830016dc70 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:1f:40:40:98:6d\",\"name\":\"8758e3eba29b21f\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/607aaaef-597a-4da2-ba00-bf04249044c8\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 02:58:36.630511 3885 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:58:36.682984 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 W0329 02:58:47.646573 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0329 02:58:47.679661 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0329 02:58:47.679736 3885 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-sm2f7 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-sm2f7: no pod IPs found I0329 02:59:01.655576 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0329 02:59:07.682713 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0329 02:59:43.644232 3885 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0329 03:00:00.179883 3885 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29579220-rh42q for pod on switch ip-10-0-2-173.ec2.internal I0329 03:00:00.180051 3885 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q I0329 03:00:00.196605 3885 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q] pod update took 16.587371ms I0329 03:00:00.196631 3885 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q] addLogicalPort annotation time took 16.619657ms I0329 03:00:00.197970 3885 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q] addLogicalPort took 18.115583ms, libovsdb time 965.896µs I0329 03:00:00.559083 3885 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q 00ff82330ed63276d2b41d8654307fbc2042ed62f60b567a87ac52ade36c4c6f network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q 00ff82330ed63276d2b41d8654307fbc2042ed62f60b567a87ac52ade36c4c6f network default NAD default] I0329 03:00:00.567678 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29579220-rh42q, hostIfaceName: 00ff82330ed6327, network: default, NAD default, SandboxID: "00ff82330ed63276d2b41d8654307fbc2042ed62f60b567a87ac52ade36c4c6f", PCI device ID: , UID: "80fb3294-c102-44f2-ab55-006e2fbab1b4", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0329 03:00:00.617559 3885 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q 00ff82330ed63276d2b41d8654307fbc2042ed62f60b567a87ac52ade36c4c6f network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q 00ff82330ed63276d2b41d8654307fbc2042ed62f60b567a87ac52ade36c4c6f network default NAD default], result "{\"interfaces\":[{\"mac\":\"ae:d2:c6:f9:d7:bf\",\"name\":\"00ff82330ed6327\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/23496a75-5700-46e2-9025-e1ef159a0947\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:00:04.011394 3885 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q 00ff82330ed63276d2b41d8654307fbc2042ed62f60b567a87ac52ade36c4c6f network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q 00ff82330ed63276d2b41d8654307fbc2042ed62f60b567a87ac52ade36c4c6f network default NAD default] I0329 03:00:04.085347 3885 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q 00ff82330ed63276d2b41d8654307fbc2042ed62f60b567a87ac52ade36c4c6f network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q 00ff82330ed63276d2b41d8654307fbc2042ed62f60b567a87ac52ade36c4c6f network default NAD default], result "{}", err I0329 03:00:04.128258 3885 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:00:04.128298 3885 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q I0329 03:00:04.129519 3885 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q, ips: 10.128.0.33 I0329 03:00:04.129547 3885 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:00:05.024454 3885 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:00:05.024526 3885 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29579220-rh42q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:00:05.676815 3885 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0329 03:00:20.659578 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0329 03:00:34.555093 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0329 03:01:02.865044 3885 namespace.go:98] [tsf] adding namespace I0329 03:01:02.865880 3885 namespace.go:102] [tsf] adding namespace took 816.566µs I0329 03:01:02.889306 3885 namespace.go:142] [tsf] updating namespace I0329 03:01:02.904565 3885 namespace.go:142] [tsf] updating namespace I0329 03:01:06.688243 3885 namespace.go:98] [rhtpa-operator] adding namespace I0329 03:01:06.689329 3885 namespace.go:102] [rhtpa-operator] adding namespace took 1.06302ms I0329 03:01:06.703098 3885 namespace.go:98] [tsf-tpa] adding namespace I0329 03:01:06.703187 3885 namespace.go:98] [konflux-operator] adding namespace I0329 03:01:06.703275 3885 namespace.go:98] [tsf-tas] adding namespace I0329 03:01:06.703524 3885 namespace.go:98] [tsf-keycloak] adding namespace I0329 03:01:06.703566 3885 namespace.go:98] [konflux-ui] adding namespace I0329 03:01:06.703685 3885 namespace.go:98] [rhbk-operator] adding namespace I0329 03:01:06.703842 3885 namespace.go:98] [cert-manager-operator] adding namespace I0329 03:01:06.703978 3885 namespace.go:102] [tsf-tpa] adding namespace took 866.621µs I0329 03:01:06.704717 3885 namespace.go:102] [konflux-operator] adding namespace took 1.509317ms I0329 03:01:06.705371 3885 namespace.go:102] [tsf-tas] adding namespace took 2.0804ms I0329 03:01:06.705958 3885 namespace.go:102] [tsf-keycloak] adding namespace took 2.414097ms I0329 03:01:06.706501 3885 namespace.go:102] [konflux-ui] adding namespace took 2.921153ms I0329 03:01:06.706964 3885 namespace.go:102] [rhbk-operator] adding namespace took 3.258808ms I0329 03:01:06.707556 3885 namespace.go:102] [cert-manager-operator] adding namespace took 3.702784ms I0329 03:01:06.725687 3885 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:01:06.746459 3885 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:01:06.761555 3885 namespace.go:142] [tsf-tpa] updating namespace I0329 03:01:06.762486 3885 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:01:06.763511 3885 namespace.go:142] [tsf-tas] updating namespace I0329 03:01:06.771516 3885 namespace.go:142] [konflux-operator] updating namespace I0329 03:01:06.782560 3885 namespace.go:142] [tsf-tpa] updating namespace I0329 03:01:06.782601 3885 namespace.go:142] [konflux-ui] updating namespace I0329 03:01:06.798703 3885 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:01:06.798832 3885 namespace.go:142] [rhbk-operator] updating namespace I0329 03:01:06.812655 3885 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:01:06.840656 3885 namespace.go:142] [tsf-tas] updating namespace I0329 03:01:06.856644 3885 namespace.go:142] [tsf-tas] updating namespace I0329 03:01:06.879742 3885 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:01:06.897119 3885 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:01:06.905219 3885 namespace.go:142] [tsf-tpa] updating namespace I0329 03:01:06.907337 3885 namespace.go:142] [konflux-ui] updating namespace I0329 03:01:06.925546 3885 namespace.go:142] [konflux-ui] updating namespace I0329 03:01:06.944664 3885 namespace.go:142] [rhbk-operator] updating namespace I0329 03:01:06.965515 3885 namespace.go:142] [rhbk-operator] updating namespace I0329 03:01:06.970549 3885 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:01:06.988996 3885 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:01:06.992601 3885 namespace.go:142] [konflux-operator] updating namespace I0329 03:01:07.006054 3885 namespace.go:142] [konflux-operator] updating namespace I0329 03:01:07.559575 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=646 I0329 03:01:07.963254 3885 namespace.go:142] [tsf-tpa] updating namespace I0329 03:01:07.963350 3885 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:01:08.649498 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0329 03:01:09.632204 3885 base_network_controller_pods.go:487] [default/openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx] creating logical port openshift-marketplace_867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:09.632322 3885 kube.go:256] Updating pod openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx I0329 03:01:09.646618 3885 pod.go:62] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx] pod update took 14.325946ms I0329 03:01:09.646639 3885 base_network_controller_pods.go:937] [default/openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx] addLogicalPort annotation time took 14.353404ms I0329 03:01:09.647856 3885 pods.go:271] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx] addLogicalPort took 15.665787ms, libovsdb time 873.854µs I0329 03:01:09.756145 3885 base_network_controller_pods.go:487] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5] creating logical port openshift-marketplace_ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:09.756272 3885 kube.go:256] Updating pod openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 I0329 03:01:09.769174 3885 pod.go:62] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5] pod update took 12.957237ms I0329 03:01:09.769192 3885 base_network_controller_pods.go:937] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5] addLogicalPort annotation time took 12.978373ms I0329 03:01:09.770214 3885 pods.go:271] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5] addLogicalPort took 14.081192ms, libovsdb time 682.378µs I0329 03:01:09.990622 3885 cni.go:403] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx e9eec40e6b0dfc5bf4ff8a8c3107f2ba41966af87d486ffec86c7514a14bf6cb network default NAD default] ADD starting CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx e9eec40e6b0dfc5bf4ff8a8c3107f2ba41966af87d486ffec86c7514a14bf6cb network default NAD default] I0329 03:01:09.999965 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx, hostIfaceName: e9eec40e6b0dfc5, network: default, NAD default, SandboxID: "e9eec40e6b0dfc5bf4ff8a8c3107f2ba41966af87d486ffec86c7514a14bf6cb", PCI device ID: , UID: "8049864d-0c10-4007-ba0e-7de925a50463", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0329 03:01:10.043275 3885 cni.go:424] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx e9eec40e6b0dfc5bf4ff8a8c3107f2ba41966af87d486ffec86c7514a14bf6cb network default NAD default] ADD finished CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx e9eec40e6b0dfc5bf4ff8a8c3107f2ba41966af87d486ffec86c7514a14bf6cb network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:c8:93:32:54:95\",\"name\":\"e9eec40e6b0dfc5\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1670df50-4918-4b75-9353-aea99177d6cb\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:10.115827 3885 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 4e4c0becc46d920154e6836dd3538c9afd3f4f9a8c9aff662f2d0de6c2ca0f65 network default NAD default] ADD starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 4e4c0becc46d920154e6836dd3538c9afd3f4f9a8c9aff662f2d0de6c2ca0f65 network default NAD default] I0329 03:01:10.124920 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5, hostIfaceName: 4e4c0becc46d920, network: default, NAD default, SandboxID: "4e4c0becc46d920154e6836dd3538c9afd3f4f9a8c9aff662f2d0de6c2ca0f65", PCI device ID: , UID: "629e4ccf-04b9-4079-b3d8-554c67c483b9", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0329 03:01:10.168591 3885 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 4e4c0becc46d920154e6836dd3538c9afd3f4f9a8c9aff662f2d0de6c2ca0f65 network default NAD default] ADD finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 4e4c0becc46d920154e6836dd3538c9afd3f4f9a8c9aff662f2d0de6c2ca0f65 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:60:1f:ed:e0:e6\",\"name\":\"4e4c0becc46d920\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d10db0cb-102e-4a9c-bdca-c1005bc414ac\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:12.680091 3885 base_network_controller_pods.go:487] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv] creating logical port openshift-marketplace_925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:12.680278 3885 kube.go:256] Updating pod openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv I0329 03:01:12.694306 3885 pod.go:62] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv] pod update took 14.088381ms I0329 03:01:12.694326 3885 base_network_controller_pods.go:937] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv] addLogicalPort annotation time took 14.113118ms I0329 03:01:12.695751 3885 pods.go:271] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv] addLogicalPort took 15.686898ms, libovsdb time 1.011208ms I0329 03:01:13.040947 3885 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv 5b3f58bc87faffc3a46a0644906b3d312fed18e405fc9dc8246996638a94eec8 network default NAD default] ADD starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv 5b3f58bc87faffc3a46a0644906b3d312fed18e405fc9dc8246996638a94eec8 network default NAD default] I0329 03:01:13.049480 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv, hostIfaceName: 5b3f58bc87faffc, network: default, NAD default, SandboxID: "5b3f58bc87faffc3a46a0644906b3d312fed18e405fc9dc8246996638a94eec8", PCI device ID: , UID: "33459d7e-8e13-48d7-8532-dca5cb0f1774", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0329 03:01:13.092846 3885 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv 5b3f58bc87faffc3a46a0644906b3d312fed18e405fc9dc8246996638a94eec8 network default NAD default] ADD finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv 5b3f58bc87faffc3a46a0644906b3d312fed18e405fc9dc8246996638a94eec8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2a:7e:fb:e3:68:d8\",\"name\":\"5b3f58bc87faffc\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c4248f3c-b6fb-4765-b09d-31d8b7d20620\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:13.345990 3885 cni.go:403] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx e9eec40e6b0dfc5bf4ff8a8c3107f2ba41966af87d486ffec86c7514a14bf6cb network default NAD default] DEL starting CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx e9eec40e6b0dfc5bf4ff8a8c3107f2ba41966af87d486ffec86c7514a14bf6cb network default NAD default] I0329 03:01:13.437230 3885 cni.go:424] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx e9eec40e6b0dfc5bf4ff8a8c3107f2ba41966af87d486ffec86c7514a14bf6cb network default NAD default] DEL finished CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx e9eec40e6b0dfc5bf4ff8a8c3107f2ba41966af87d486ffec86c7514a14bf6cb network default NAD default], result "{}", err I0329 03:01:13.554584 3885 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:13.554636 3885 pods.go:176] Deleting pod: openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx I0329 03:01:13.556088 3885 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx, ips: 10.128.0.34 I0329 03:01:13.556117 3885 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:13.556139 3885 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:13.556146 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:13.556364 3885 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:13.556383 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:14.465561 3885 base_network_controller_pods.go:487] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n] creating logical port openshift-marketplace_3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:14.465684 3885 kube.go:256] Updating pod openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n I0329 03:01:14.479240 3885 pod.go:62] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n] pod update took 13.595187ms I0329 03:01:14.479269 3885 base_network_controller_pods.go:937] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n] addLogicalPort annotation time took 13.631788ms I0329 03:01:14.480573 3885 pods.go:271] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n] addLogicalPort took 15.032833ms, libovsdb time 929.777µs I0329 03:01:14.580715 3885 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:14.580770 3885 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:14.580788 3885 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:14.580796 3885 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf97pzx of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:15.263850 3885 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n 30c5f5ad0f039b8bc2df2314cfab3acba11bbb215a3678293eec0b04c86a18d8 network default NAD default] ADD starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n 30c5f5ad0f039b8bc2df2314cfab3acba11bbb215a3678293eec0b04c86a18d8 network default NAD default] I0329 03:01:15.272331 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n, hostIfaceName: 30c5f5ad0f039b8, network: default, NAD default, SandboxID: "30c5f5ad0f039b8bc2df2314cfab3acba11bbb215a3678293eec0b04c86a18d8", PCI device ID: , UID: "571d928c-6151-4637-95aa-b6eafbd460da", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0329 03:01:15.316557 3885 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n 30c5f5ad0f039b8bc2df2314cfab3acba11bbb215a3678293eec0b04c86a18d8 network default NAD default] ADD finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n 30c5f5ad0f039b8bc2df2314cfab3acba11bbb215a3678293eec0b04c86a18d8 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:df:77:0b:c2:98\",\"name\":\"30c5f5ad0f039b8\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e5ded21a-399a-4ede-9b74-560a38317fee\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:15.358802 3885 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 4e4c0becc46d920154e6836dd3538c9afd3f4f9a8c9aff662f2d0de6c2ca0f65 network default NAD default] DEL starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 4e4c0becc46d920154e6836dd3538c9afd3f4f9a8c9aff662f2d0de6c2ca0f65 network default NAD default] I0329 03:01:15.431652 3885 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 4e4c0becc46d920154e6836dd3538c9afd3f4f9a8c9aff662f2d0de6c2ca0f65 network default NAD default] DEL finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 4e4c0becc46d920154e6836dd3538c9afd3f4f9a8c9aff662f2d0de6c2ca0f65 network default NAD default], result "{}", err I0329 03:01:15.595904 3885 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:15.595944 3885 pods.go:176] Deleting pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 I0329 03:01:15.597420 3885 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5, ips: 10.128.0.35 I0329 03:01:15.597444 3885 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:15.597487 3885 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:15.597498 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:15.597693 3885 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:15.597709 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:15.903260 3885 base_network_controller_pods.go:487] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d] creating logical port openshift-marketplace_30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:15.903360 3885 kube.go:256] Updating pod openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d I0329 03:01:15.917223 3885 pod.go:62] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d] pod update took 13.896721ms I0329 03:01:15.917244 3885 base_network_controller_pods.go:937] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d] addLogicalPort annotation time took 13.922595ms I0329 03:01:15.918574 3885 pods.go:271] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d] addLogicalPort took 15.325279ms, libovsdb time 968.941µs I0329 03:01:16.003440 3885 base_network_controller_pods.go:487] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9] creating logical port openshift-marketplace_18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:16.003550 3885 kube.go:256] Updating pod openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 I0329 03:01:16.017084 3885 pod.go:62] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9] pod update took 13.554691ms I0329 03:01:16.017108 3885 base_network_controller_pods.go:937] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9] addLogicalPort annotation time took 13.582589ms I0329 03:01:16.018375 3885 pods.go:271] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9] addLogicalPort took 14.943238ms, libovsdb time 791.616µs I0329 03:01:16.262863 3885 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d 58466a926062311be7adce45dc132914309b21d1db5fd98235af1f9438b58afa network default NAD default] ADD starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d 58466a926062311be7adce45dc132914309b21d1db5fd98235af1f9438b58afa network default NAD default] I0329 03:01:16.271385 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d, hostIfaceName: 58466a926062311, network: default, NAD default, SandboxID: "58466a926062311be7adce45dc132914309b21d1db5fd98235af1f9438b58afa", PCI device ID: , UID: "b7b739a0-e7f3-4fef-9538-290f4d7ec530", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0329 03:01:16.314547 3885 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d 58466a926062311be7adce45dc132914309b21d1db5fd98235af1f9438b58afa network default NAD default] ADD finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d 58466a926062311be7adce45dc132914309b21d1db5fd98235af1f9438b58afa network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:5f:ec:36:33:0e\",\"name\":\"58466a926062311\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/22a31430-80d5-4122-9d17-8e8f2e10acf4\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:16.372155 3885 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 d122f670e566485d41e11e8536fa53ab09860f4787989cf790292d13c9f68b32 network default NAD default] ADD starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 d122f670e566485d41e11e8536fa53ab09860f4787989cf790292d13c9f68b32 network default NAD default] I0329 03:01:16.381351 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9, hostIfaceName: d122f670e566485, network: default, NAD default, SandboxID: "d122f670e566485d41e11e8536fa53ab09860f4787989cf790292d13c9f68b32", PCI device ID: , UID: "1c370ccf-3ef0-4e76-b3a9-01260d0e01d6", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0329 03:01:16.429340 3885 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 d122f670e566485d41e11e8536fa53ab09860f4787989cf790292d13c9f68b32 network default NAD default] ADD finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 d122f670e566485d41e11e8536fa53ab09860f4787989cf790292d13c9f68b32 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:11:79:8d:83:9f\",\"name\":\"d122f670e566485\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/59eae836-678a-4c1d-923e-11de6f55b050\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:16.624071 3885 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:16.624102 3885 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:16.624116 3885 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:16.624135 3885 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba8xmgd5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:18.394184 3885 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv 5b3f58bc87faffc3a46a0644906b3d312fed18e405fc9dc8246996638a94eec8 network default NAD default] DEL starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv 5b3f58bc87faffc3a46a0644906b3d312fed18e405fc9dc8246996638a94eec8 network default NAD default] I0329 03:01:18.483442 3885 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv 5b3f58bc87faffc3a46a0644906b3d312fed18e405fc9dc8246996638a94eec8 network default NAD default] DEL finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv 5b3f58bc87faffc3a46a0644906b3d312fed18e405fc9dc8246996638a94eec8 network default NAD default], result "{}", err I0329 03:01:18.608623 3885 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:18.608664 3885 pods.go:176] Deleting pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv I0329 03:01:18.610050 3885 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv, ips: 10.128.0.36 I0329 03:01:18.610078 3885 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:18.610100 3885 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:18.610108 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:18.610295 3885 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:18.610314 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:19.640827 3885 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:19.640854 3885 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:19.640871 3885 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:19.640881 3885 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5vqcxv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:20.417376 3885 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n 30c5f5ad0f039b8bc2df2314cfab3acba11bbb215a3678293eec0b04c86a18d8 network default NAD default] DEL starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n 30c5f5ad0f039b8bc2df2314cfab3acba11bbb215a3678293eec0b04c86a18d8 network default NAD default] I0329 03:01:20.490032 3885 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n 30c5f5ad0f039b8bc2df2314cfab3acba11bbb215a3678293eec0b04c86a18d8 network default NAD default] DEL finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n 30c5f5ad0f039b8bc2df2314cfab3acba11bbb215a3678293eec0b04c86a18d8 network default NAD default], result "{}", err I0329 03:01:20.538181 3885 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:20.538214 3885 pods.go:176] Deleting pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n I0329 03:01:20.539776 3885 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n, ips: 10.128.0.37 I0329 03:01:20.539806 3885 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:20.539826 3885 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:20.539833 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:20.540043 3885 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:20.540060 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:21.427308 3885 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 d122f670e566485d41e11e8536fa53ab09860f4787989cf790292d13c9f68b32 network default NAD default] DEL starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 d122f670e566485d41e11e8536fa53ab09860f4787989cf790292d13c9f68b32 network default NAD default] I0329 03:01:21.427329 3885 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d 58466a926062311be7adce45dc132914309b21d1db5fd98235af1f9438b58afa network default NAD default] DEL starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d 58466a926062311be7adce45dc132914309b21d1db5fd98235af1f9438b58afa network default NAD default] I0329 03:01:21.495188 3885 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 d122f670e566485d41e11e8536fa53ab09860f4787989cf790292d13c9f68b32 network default NAD default] DEL finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 d122f670e566485d41e11e8536fa53ab09860f4787989cf790292d13c9f68b32 network default NAD default], result "{}", err I0329 03:01:21.526831 3885 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d 58466a926062311be7adce45dc132914309b21d1db5fd98235af1f9438b58afa network default NAD default] DEL finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d 58466a926062311be7adce45dc132914309b21d1db5fd98235af1f9438b58afa network default NAD default], result "{}", err I0329 03:01:21.544225 3885 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.544254 3885 pods.go:176] Deleting pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 I0329 03:01:21.545776 3885 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9, ips: 10.128.0.39 I0329 03:01:21.545808 3885 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.545834 3885 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.545842 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:21.546092 3885 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.546112 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:21.572773 3885 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:21.572832 3885 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:21.572856 3885 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:21.572870 3885 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d12pd7n of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:21.585761 3885 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.585786 3885 pods.go:176] Deleting pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d I0329 03:01:21.586940 3885 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d, ips: 10.128.0.38 I0329 03:01:21.586966 3885 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.586984 3885 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.586995 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0329 03:01:21.587154 3885 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0329 03:01:21.587169 3885 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0329 03:01:22.572290 3885 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.572320 3885 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.572343 3885 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.572355 3885 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7nczn9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.613909 3885 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.613930 3885 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.613943 3885 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:22.613951 3885 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafs5l9d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:01:23.507674 3885 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhxnr] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-nhxnr for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:23.507791 3885 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhxnr I0329 03:01:23.523139 3885 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhxnr] pod update took 15.373586ms I0329 03:01:23.523163 3885 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhxnr] addLogicalPort annotation time took 15.402523ms I0329 03:01:23.524394 3885 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhxnr] addLogicalPort took 16.741891ms, libovsdb time 848.528µs I0329 03:01:23.867196 3885 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhxnr 794b63d5a6a965279d2c9df041ba300cb8508195c7ab99a85d66233a48b66555 network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhxnr 794b63d5a6a965279d2c9df041ba300cb8508195c7ab99a85d66233a48b66555 network default NAD default] I0329 03:01:23.875458 3885 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-nhxnr, hostIfaceName: 794b63d5a6a9652, network: default, NAD default, SandboxID: "794b63d5a6a965279d2c9df041ba300cb8508195c7ab99a85d66233a48b66555", PCI device ID: , UID: "f5bb7e8f-6d4d-4417-aa03-f3d14db11988", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0329 03:01:23.917826 3885 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhxnr 794b63d5a6a965279d2c9df041ba300cb8508195c7ab99a85d66233a48b66555 network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-nhxnr 794b63d5a6a965279d2c9df041ba300cb8508195c7ab99a85d66233a48b66555 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:c9:8d:d6:5f:9b\",\"name\":\"794b63d5a6a9652\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ec54f971-ac30-4239-be0d-ddb5d011892c\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 03:01:27.464120 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0329 03:01:28.743060 3885 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-4px5p] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-4px5p for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:28.743202 3885 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-4px5p I0329 03:01:28.758870 3885 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-4px5p] pod update took 15.69534ms I0329 03:01:28.758893 3885 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-4px5p] addLogicalPort annotation time took 15.7204ms I0329 03:01:28.760108 3885 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-4px5p] addLogicalPort took 17.05861ms, libovsdb time 882.941µs I0329 03:01:29.105018 3885 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-4px5p f180155b384c38f5f5ff354652f29061ba4be229021edc6f0a5e465783c0603d network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-4px5p f180155b384c38f5f5ff354652f29061ba4be229021edc6f0a5e465783c0603d network default NAD default] I0329 03:01:29.113728 3885 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-4px5p, hostIfaceName: f180155b384c38f, network: default, NAD default, SandboxID: "f180155b384c38f5f5ff354652f29061ba4be229021edc6f0a5e465783c0603d", PCI device ID: , UID: "770a28ad-aae5-4cb9-b55b-a22002f77c2b", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0329 03:01:29.157729 3885 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-4px5p f180155b384c38f5f5ff354652f29061ba4be229021edc6f0a5e465783c0603d network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-4px5p f180155b384c38f5f5ff354652f29061ba4be229021edc6f0a5e465783c0603d network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:3d:79:15:7e:df\",\"name\":\"f180155b384c38f\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e0023911-dc2f-4f73-846f-e58dc5750a40\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 03:01:31.972147 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0329 03:01:32.022954 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0329 03:01:32.768702 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0329 03:01:33.995641 3885 namespace.go:98] [cert-manager] adding namespace I0329 03:01:33.996600 3885 namespace.go:102] [cert-manager] adding namespace took 930.509µs I0329 03:01:34.033983 3885 namespace.go:142] [cert-manager] updating namespace I0329 03:01:34.055146 3885 namespace.go:142] [cert-manager] updating namespace I0329 03:01:34.609894 3885 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-8468f77bc5-h5mb9] creating logical port rhbk-operator_rhbk-operator-8468f77bc5-h5mb9 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:34.610038 3885 kube.go:256] Updating pod rhbk-operator/rhbk-operator-8468f77bc5-h5mb9 I0329 03:01:34.641618 3885 pod.go:62] [rhbk-operator/rhbk-operator-8468f77bc5-h5mb9] pod update took 31.611829ms I0329 03:01:34.641637 3885 base_network_controller_pods.go:937] [default/rhbk-operator/rhbk-operator-8468f77bc5-h5mb9] addLogicalPort annotation time took 31.635173ms I0329 03:01:34.642869 3885 pods.go:271] [rhbk-operator/rhbk-operator-8468f77bc5-h5mb9] addLogicalPort took 32.994008ms, libovsdb time 873.44µs I0329 03:01:34.992066 3885 cni.go:403] [rhbk-operator/rhbk-operator-8468f77bc5-h5mb9 23ab022243955a2647576d892306b84b9d5ebe40833eec7d349c96cf5a183a5f network default NAD default] ADD starting CNI request [rhbk-operator/rhbk-operator-8468f77bc5-h5mb9 23ab022243955a2647576d892306b84b9d5ebe40833eec7d349c96cf5a183a5f network default NAD default] I0329 03:01:35.001764 3885 helper_linux.go:450] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-8468f77bc5-h5mb9, hostIfaceName: 23ab022243955a2, network: default, NAD default, SandboxID: "23ab022243955a2647576d892306b84b9d5ebe40833eec7d349c96cf5a183a5f", PCI device ID: , UID: "35677e15-15cc-41a1-9264-972d5cae72fc", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0329 03:01:35.049054 3885 cni.go:424] [rhbk-operator/rhbk-operator-8468f77bc5-h5mb9 23ab022243955a2647576d892306b84b9d5ebe40833eec7d349c96cf5a183a5f network default NAD default] ADD finished CNI request [rhbk-operator/rhbk-operator-8468f77bc5-h5mb9 23ab022243955a2647576d892306b84b9d5ebe40833eec7d349c96cf5a183a5f network default NAD default], result "{\"interfaces\":[{\"mac\":\"d6:28:71:de:2b:ff\",\"name\":\"23ab022243955a2\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/67767ef5-4de0-4ef0-bc1e-b2e2b5b1bce5\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:35.813622 3885 namespace.go:142] [openshift-operators] updating namespace I0329 03:01:35.825769 3885 namespace.go:142] [openshift-operators] updating namespace W0329 03:01:38.492403 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0329 03:01:39.042252 3885 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5b4bf68748-hz4p5] creating logical port cert-manager_cert-manager-cainjector-5b4bf68748-hz4p5 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:39.042446 3885 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5b4bf68748-hz4p5 I0329 03:01:39.056324 3885 pod.go:62] [cert-manager/cert-manager-cainjector-5b4bf68748-hz4p5] pod update took 13.907102ms I0329 03:01:39.056348 3885 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-cainjector-5b4bf68748-hz4p5] addLogicalPort annotation time took 13.952827ms I0329 03:01:39.057770 3885 pods.go:271] [cert-manager/cert-manager-cainjector-5b4bf68748-hz4p5] addLogicalPort took 15.539804ms, libovsdb time 961.764µs W0329 03:01:39.902657 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0329 03:01:39.902727 3885 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0329 03:01:40.216372 3885 cni.go:403] [cert-manager/cert-manager-cainjector-5b4bf68748-hz4p5 12f3097ecaca226f71d2217d51da57200f5bac3422c980414b51561b85005f5c network default NAD default] ADD starting CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-hz4p5 12f3097ecaca226f71d2217d51da57200f5bac3422c980414b51561b85005f5c network default NAD default] I0329 03:01:40.226057 3885 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5b4bf68748-hz4p5, hostIfaceName: 12f3097ecaca226, network: default, NAD default, SandboxID: "12f3097ecaca226f71d2217d51da57200f5bac3422c980414b51561b85005f5c", PCI device ID: , UID: "9f963e44-c896-4b63-9461-ef2f585ae966", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0329 03:01:40.271531 3885 cni.go:424] [cert-manager/cert-manager-cainjector-5b4bf68748-hz4p5 12f3097ecaca226f71d2217d51da57200f5bac3422c980414b51561b85005f5c network default NAD default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5b4bf68748-hz4p5 12f3097ecaca226f71d2217d51da57200f5bac3422c980414b51561b85005f5c network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:7b:75:58:a0:8c\",\"name\":\"12f3097ecaca226\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ea7107e7-a88e-4f9c-aa78-f147e5a26c84\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:42.596432 3885 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-57v9s] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-57v9s for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:42.596566 3885 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-57v9s I0329 03:01:42.611873 3885 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-57v9s] pod update took 15.33473ms I0329 03:01:42.611893 3885 base_network_controller_pods.go:937] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-57v9s] addLogicalPort annotation time took 15.36309ms I0329 03:01:42.613184 3885 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-57v9s] addLogicalPort took 16.767929ms, libovsdb time 820.971µs I0329 03:01:43.070625 3885 cni.go:403] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-57v9s 4710af04d2c6b7d08e2645df57e51d09b832926987555b4ac646ce157c7b1009 network default NAD default] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-57v9s 4710af04d2c6b7d08e2645df57e51d09b832926987555b4ac646ce157c7b1009 network default NAD default] I0329 03:01:43.081015 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-57v9s, hostIfaceName: 4710af04d2c6b7d, network: default, NAD default, SandboxID: "4710af04d2c6b7d08e2645df57e51d09b832926987555b4ac646ce157c7b1009", PCI device ID: , UID: "1c9112ba-fe72-405c-9266-06d539099fbb", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0329 03:01:43.337435 3885 cni.go:424] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-57v9s 4710af04d2c6b7d08e2645df57e51d09b832926987555b4ac646ce157c7b1009 network default NAD default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-57v9s 4710af04d2c6b7d08e2645df57e51d09b832926987555b4ac646ce157c7b1009 network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:bf:8b:cc:4c:4a\",\"name\":\"4710af04d2c6b7d\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2622d7a9-e3da-4b51-af8b-3cfb8514ffc8\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:46.295337 3885 namespace.go:98] [trusted-artifact-signer] adding namespace I0329 03:01:46.296222 3885 namespace.go:102] [trusted-artifact-signer] adding namespace took 862.232µs I0329 03:01:46.327638 3885 namespace.go:142] [trusted-artifact-signer] updating namespace I0329 03:01:46.349550 3885 namespace.go:142] [trusted-artifact-signer] updating namespace I0329 03:01:46.529535 3885 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-fgffs] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-fgffs for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:46.529676 3885 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-fgffs I0329 03:01:46.542161 3885 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-fgffs] pod update took 12.517088ms I0329 03:01:46.542191 3885 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-fgffs] addLogicalPort annotation time took 12.549472ms I0329 03:01:46.545081 3885 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-fgffs] addLogicalPort took 15.56087ms, libovsdb time 2.363672ms W0329 03:01:46.652333 3885 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0329 03:01:46.902663 3885 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-fgffs 9b3b93d5886e6bad0809494b619c12c06b0fb26956b5b909b279cddd6ffad67f network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-fgffs 9b3b93d5886e6bad0809494b619c12c06b0fb26956b5b909b279cddd6ffad67f network default NAD default] I0329 03:01:46.911166 3885 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-fgffs, hostIfaceName: 9b3b93d5886e6ba, network: default, NAD default, SandboxID: "9b3b93d5886e6bad0809494b619c12c06b0fb26956b5b909b279cddd6ffad67f", PCI device ID: , UID: "0e226bcc-ad0e-48bd-b7bf-eeaa8d82e0f1", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0329 03:01:46.957514 3885 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-fgffs 9b3b93d5886e6bad0809494b619c12c06b0fb26956b5b909b279cddd6ffad67f network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-fgffs 9b3b93d5886e6bad0809494b619c12c06b0fb26956b5b909b279cddd6ffad67f network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:41:d7:f8:7c:6a\",\"name\":\"9b3b93d5886e6ba\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb8c1dda-0051-487b-8242-280cc460bec7\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:01:49.347704 3885 namespace.go:142] [openshift-operators] updating namespace I0329 03:01:49.709985 3885 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6684678d86-9td75] creating logical port openshift-operators_openshift-pipelines-operator-6684678d86-9td75 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:01:49.710106 3885 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6684678d86-9td75 I0329 03:01:49.752769 3885 pod.go:62] [openshift-operators/openshift-pipelines-operator-6684678d86-9td75] pod update took 42.691734ms I0329 03:01:49.752800 3885 base_network_controller_pods.go:937] [default/openshift-operators/openshift-pipelines-operator-6684678d86-9td75] addLogicalPort annotation time took 42.725627ms I0329 03:01:49.754405 3885 pods.go:271] [openshift-operators/openshift-pipelines-operator-6684678d86-9td75] addLogicalPort took 44.436236ms, libovsdb time 1.100112ms I0329 03:01:50.074925 3885 cni.go:403] [openshift-operators/openshift-pipelines-operator-6684678d86-9td75 0542ea8433d744a30973c4d9240a06819f42d7d4a5aa45b7fbc7e0fa63e542b7 network default NAD default] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-9td75 0542ea8433d744a30973c4d9240a06819f42d7d4a5aa45b7fbc7e0fa63e542b7 network default NAD default] I0329 03:01:50.084370 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6684678d86-9td75, hostIfaceName: 0542ea8433d744a, network: default, NAD default, SandboxID: "0542ea8433d744a30973c4d9240a06819f42d7d4a5aa45b7fbc7e0fa63e542b7", PCI device ID: , UID: "4107b4cd-0a11-4282-a6a9-3372e7eb41f2", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0329 03:01:50.129558 3885 cni.go:424] [openshift-operators/openshift-pipelines-operator-6684678d86-9td75 0542ea8433d744a30973c4d9240a06819f42d7d4a5aa45b7fbc7e0fa63e542b7 network default NAD default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6684678d86-9td75 0542ea8433d744a30973c4d9240a06819f42d7d4a5aa45b7fbc7e0fa63e542b7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:29:c2:3a:ad:1d\",\"name\":\"0542ea8433d744a\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ad7e39ba-79cc-4825-aa1b-e2edc088e4ec\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 03:01:57.601907 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0329 03:02:04.167383 3885 namespace.go:98] [openshift-pipelines] adding namespace I0329 03:02:04.168249 3885 namespace.go:102] [openshift-pipelines] adding namespace took 844.045µs I0329 03:02:04.193248 3885 namespace.go:142] [openshift-pipelines] updating namespace I0329 03:02:04.882626 3885 namespace.go:142] [cert-manager] updating namespace I0329 03:02:05.081237 3885 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:02:05.484309 3885 namespace.go:142] [konflux-operator] updating namespace I0329 03:02:05.684862 3885 namespace.go:142] [konflux-ui] updating namespace I0329 03:02:05.887050 3885 namespace.go:142] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] updating namespace I0329 03:02:06.290023 3885 namespace.go:142] [rhbk-operator] updating namespace I0329 03:02:06.488812 3885 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:02:06.688192 3885 namespace.go:142] [trusted-artifact-signer] updating namespace I0329 03:02:06.887310 3885 namespace.go:142] [tsf] updating namespace I0329 03:02:07.089930 3885 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:02:07.292023 3885 namespace.go:142] [tsf-tas] updating namespace I0329 03:02:07.490502 3885 namespace.go:142] [tsf-tpa] updating namespace I0329 03:02:07.628901 3885 namespace.go:142] [cert-manager] updating namespace I0329 03:02:07.642695 3885 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:02:07.655253 3885 namespace.go:142] [default] updating namespace I0329 03:02:07.669823 3885 namespace.go:142] [konflux-operator] updating namespace I0329 03:02:07.682855 3885 namespace.go:142] [konflux-ui] updating namespace I0329 03:02:07.691794 3885 namespace.go:142] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] updating namespace I0329 03:02:07.703957 3885 namespace.go:142] [openshift] updating namespace I0329 03:02:07.716788 3885 namespace.go:142] [rhbk-operator] updating namespace I0329 03:02:07.730416 3885 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:02:07.740068 3885 namespace.go:142] [trusted-artifact-signer] updating namespace I0329 03:02:07.751313 3885 namespace.go:142] [tsf] updating namespace I0329 03:02:07.760882 3885 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:02:07.777074 3885 namespace.go:142] [tsf-tas] updating namespace I0329 03:02:07.792487 3885 namespace.go:142] [tsf-tpa] updating namespace I0329 03:02:07.875708 3885 namespace.go:142] [cert-manager] updating namespace I0329 03:02:07.976645 3885 namespace.go:142] [cert-manager-operator] updating namespace I0329 03:02:08.072840 3885 namespace.go:142] [default] updating namespace I0329 03:02:08.173961 3885 namespace.go:142] [konflux-operator] updating namespace I0329 03:02:08.278017 3885 namespace.go:142] [konflux-ui] updating namespace I0329 03:02:08.373318 3885 namespace.go:142] [open-cluster-management-2pb44o7svvhvrbms3qmq2smscrbj81th] updating namespace I0329 03:02:08.478980 3885 namespace.go:142] [openshift] updating namespace I0329 03:02:08.573793 3885 namespace.go:142] [rhbk-operator] updating namespace I0329 03:02:08.672398 3885 namespace.go:142] [rhtpa-operator] updating namespace I0329 03:02:08.772593 3885 namespace.go:142] [trusted-artifact-signer] updating namespace I0329 03:02:08.877990 3885 namespace.go:142] [tsf] updating namespace I0329 03:02:08.978787 3885 namespace.go:142] [tsf-keycloak] updating namespace I0329 03:02:09.079206 3885 namespace.go:142] [tsf-tas] updating namespace I0329 03:02:09.189603 3885 namespace.go:142] [tsf-tpa] updating namespace I0329 03:02:09.601100 3885 namespace.go:142] [openshift-pipelines] updating namespace I0329 03:02:17.633311 3885 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt for pod on switch ip-10-0-2-173.ec2.internal I0329 03:02:17.633446 3885 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt I0329 03:02:17.647667 3885 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt] pod update took 14.251138ms I0329 03:02:17.647691 3885 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt] addLogicalPort annotation time took 14.276068ms I0329 03:02:17.648844 3885 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt] addLogicalPort took 15.547886ms, libovsdb time 769.506µs I0329 03:02:18.119277 3885 cni.go:403] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt a3cccf1ed483aa2d13ad8cc2fd638d03750b6d8f7df96541b5409b488c1721fb network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt a3cccf1ed483aa2d13ad8cc2fd638d03750b6d8f7df96541b5409b488c1721fb network default NAD default] I0329 03:02:18.128514 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt, hostIfaceName: a3cccf1ed483aa2, network: default, NAD default, SandboxID: "a3cccf1ed483aa2d13ad8cc2fd638d03750b6d8f7df96541b5409b488c1721fb", PCI device ID: , UID: "970e2b10-09fa-464b-95d2-87baec9d6524", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0329 03:02:18.178614 3885 cni.go:424] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt a3cccf1ed483aa2d13ad8cc2fd638d03750b6d8f7df96541b5409b488c1721fb network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-kk8tt a3cccf1ed483aa2d13ad8cc2fd638d03750b6d8f7df96541b5409b488c1721fb network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:db:5e:57:72:5c\",\"name\":\"a3cccf1ed483aa2\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fc408361-7ad0-4619-a78d-b4580c0b1c2f\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:02:19.456043 3885 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:02:19.456694 3885 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:02:19.478264 3885 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:02:19.478289 3885 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:02:19.486738 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0329 03:02:19.486789 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions W0329 03:02:20.894316 3885 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0329 03:02:20.894335 3885 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0329 03:02:20.894341 3885 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0329 03:02:21.331737 3885 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0329 03:02:21.331764 3885 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0329 03:02:21.331769 3885 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0329 03:02:21.923355 3885 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:21.923374 3885 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:21.923380 3885 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:21.923384 3885 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0329 03:02:31.386989 3885 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0329 03:02:31.413177 3885 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0329 03:02:40.690869 3885 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-mxfch] creating logical port openshift-pipelines_tekton-triggers-controller-68564d75ff-mxfch for pod on switch ip-10-0-2-173.ec2.internal I0329 03:02:40.690952 3885 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-68564d75ff-mxfch I0329 03:02:40.707263 3885 pod.go:62] [openshift-pipelines/tekton-triggers-controller-68564d75ff-mxfch] pod update took 16.331766ms I0329 03:02:40.707288 3885 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-controller-68564d75ff-mxfch] addLogicalPort annotation time took 16.362756ms I0329 03:02:40.708509 3885 pods.go:271] [openshift-pipelines/tekton-triggers-controller-68564d75ff-mxfch] addLogicalPort took 17.653666ms, libovsdb time 827.229µs I0329 03:02:40.815217 3885 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-k6j8f] creating logical port openshift-pipelines_tekton-triggers-webhook-55d94ff6c6-k6j8f for pod on switch ip-10-0-2-173.ec2.internal I0329 03:02:40.815303 3885 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-k6j8f I0329 03:02:40.832769 3885 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-k6j8f] pod update took 17.489688ms I0329 03:02:40.832791 3885 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-k6j8f] addLogicalPort annotation time took 17.515292ms I0329 03:02:40.833968 3885 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-k6j8f] addLogicalPort took 18.768667ms, libovsdb time 809.014µs I0329 03:02:40.884894 3885 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-bl6c7] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-8bb674894-bl6c7 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:02:40.885019 3885 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-bl6c7 I0329 03:02:40.905167 3885 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-bl6c7] pod update took 20.176529ms I0329 03:02:40.905187 3885 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-bl6c7] addLogicalPort annotation time took 20.200867ms I0329 03:02:40.906358 3885 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-bl6c7] addLogicalPort took 21.513754ms, libovsdb time 752.625µs I0329 03:02:41.056032 3885 cni.go:403] [openshift-pipelines/tekton-triggers-controller-68564d75ff-mxfch 271b1b09dcdeb9e275be73dcb0839137077fa6ce768b4fff90745784d5e64e25 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-mxfch 271b1b09dcdeb9e275be73dcb0839137077fa6ce768b4fff90745784d5e64e25 network default NAD default] I0329 03:02:41.064710 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-68564d75ff-mxfch, hostIfaceName: 271b1b09dcdeb9e, network: default, NAD default, SandboxID: "271b1b09dcdeb9e275be73dcb0839137077fa6ce768b4fff90745784d5e64e25", PCI device ID: , UID: "ff52067e-dad1-4611-ab7f-54aeb2b23ba5", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0329 03:02:41.110044 3885 cni.go:424] [openshift-pipelines/tekton-triggers-controller-68564d75ff-mxfch 271b1b09dcdeb9e275be73dcb0839137077fa6ce768b4fff90745784d5e64e25 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-68564d75ff-mxfch 271b1b09dcdeb9e275be73dcb0839137077fa6ce768b4fff90745784d5e64e25 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:78:7d:ab:61:e2\",\"name\":\"271b1b09dcdeb9e\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/42ee66ee-2941-457c-8137-00dc12f344f1\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:02:41.181122 3885 cni.go:403] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-k6j8f bf13f93e3c9e707aa442228f54cf4813f0ab98e7c6ef630c277a9842825355a6 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-k6j8f bf13f93e3c9e707aa442228f54cf4813f0ab98e7c6ef630c277a9842825355a6 network default NAD default] I0329 03:02:41.189858 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-55d94ff6c6-k6j8f, hostIfaceName: bf13f93e3c9e707, network: default, NAD default, SandboxID: "bf13f93e3c9e707aa442228f54cf4813f0ab98e7c6ef630c277a9842825355a6", PCI device ID: , UID: "bc03c855-2425-4b26-b0ab-582d62bfe57f", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0329 03:02:41.239117 3885 cni.go:424] [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-k6j8f bf13f93e3c9e707aa442228f54cf4813f0ab98e7c6ef630c277a9842825355a6 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-55d94ff6c6-k6j8f bf13f93e3c9e707aa442228f54cf4813f0ab98e7c6ef630c277a9842825355a6 network default NAD default], result "{\"interfaces\":[{\"mac\":\"66:ef:9e:32:48:1c\",\"name\":\"bf13f93e3c9e707\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5fc555ca-3293-419e-b407-6b90016f63e6\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:02:41.247593 3885 cni.go:403] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-bl6c7 1cd37ce4056e0eb14b1be40dc3e4579ccded76048e5707bcdf3f55a5a9405350 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-bl6c7 1cd37ce4056e0eb14b1be40dc3e4579ccded76048e5707bcdf3f55a5a9405350 network default NAD default] I0329 03:02:41.256195 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-8bb674894-bl6c7, hostIfaceName: 1cd37ce4056e0eb, network: default, NAD default, SandboxID: "1cd37ce4056e0eb14b1be40dc3e4579ccded76048e5707bcdf3f55a5a9405350", PCI device ID: , UID: "e82276df-de83-4b00-aa05-85b03bee81cc", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0329 03:02:41.302239 3885 cni.go:424] [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-bl6c7 1cd37ce4056e0eb14b1be40dc3e4579ccded76048e5707bcdf3f55a5a9405350 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-8bb674894-bl6c7 1cd37ce4056e0eb14b1be40dc3e4579ccded76048e5707bcdf3f55a5a9405350 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:88:5c:3b:f7:6f\",\"name\":\"1cd37ce4056e0eb\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/752f1c16-ca13-4449-9c85-040ab1459d45\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 03:02:46.935961 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0329 03:02:54.645588 3885 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:02:54.646117 3885 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:02:54.673173 3885 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:02:54.673195 3885 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:02:54.681576 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0329 03:02:54.681600 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure W0329 03:02:57.950731 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0329 03:03:01.647939 3885 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0329 03:03:04.218275 3885 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0329 03:03:04.564582 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0329 03:03:04.564582 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:03:04.564690 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0329 03:03:07.307242 3885 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:03:07.307334 3885 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0329 03:03:07.320519 3885 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 13.206904ms I0329 03:03:07.320540 3885 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 13.232675ms I0329 03:03:07.321820 3885 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 14.593117ms, libovsdb time 896.869µs I0329 03:03:16.681700 3885 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 259f44cc54618ca64bbd07025427b739d72d239e7e5b5dcad0918c48a9c23f6b network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 259f44cc54618ca64bbd07025427b739d72d239e7e5b5dcad0918c48a9c23f6b network default NAD default] I0329 03:03:16.695400 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 259f44cc54618ca, network: default, NAD default, SandboxID: "259f44cc54618ca64bbd07025427b739d72d239e7e5b5dcad0918c48a9c23f6b", PCI device ID: , UID: "50b935a5-9b8b-4f3b-be84-06e6ab818624", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0329 03:03:16.745332 3885 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 259f44cc54618ca64bbd07025427b739d72d239e7e5b5dcad0918c48a9c23f6b network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 259f44cc54618ca64bbd07025427b739d72d239e7e5b5dcad0918c48a9c23f6b network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:d8:ce:b4:ee:dc\",\"name\":\"259f44cc54618ca\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/06e0f14b-cfba-44de-bff2-8dfdead46817\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:03:23.189413 3885 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0329 03:03:24.925925 3885 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:03:24.926433 3885 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:03:25.669450 3885 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:03:25.669497 3885 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:03:25.677704 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0329 03:03:25.677734 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0329 03:03:28.699018 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0329 03:03:28.699128 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:03:28.699146 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:03:30.250579 3885 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0329 03:03:30.250602 3885 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0329 03:03:30.250607 3885 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0329 03:03:51.009359 3885 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:03:51.010011 3885 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:03:51.052547 3885 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:03:51.052568 3885 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:03:51.061776 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0329 03:03:51.061800 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0329 03:03:53.934360 3885 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-576fd589f9-jbtbd] creating logical port tsf-tas_fulcio-server-576fd589f9-jbtbd for pod on switch ip-10-0-2-173.ec2.internal I0329 03:03:53.934448 3885 kube.go:256] Updating pod tsf-tas/fulcio-server-576fd589f9-jbtbd I0329 03:03:53.948402 3885 pod.go:62] [tsf-tas/fulcio-server-576fd589f9-jbtbd] pod update took 13.97495ms I0329 03:03:53.948428 3885 base_network_controller_pods.go:937] [default/tsf-tas/fulcio-server-576fd589f9-jbtbd] addLogicalPort annotation time took 14.003795ms I0329 03:03:53.949717 3885 pods.go:271] [tsf-tas/fulcio-server-576fd589f9-jbtbd] addLogicalPort took 15.376224ms, libovsdb time 835.524µs I0329 03:03:54.005289 3885 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-mlckb-92rgz] creating logical port tsf-tas_ctlog-createtree-job-mlckb-92rgz for pod on switch ip-10-0-2-173.ec2.internal I0329 03:03:54.005362 3885 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-mlckb-92rgz I0329 03:03:54.020903 3885 pod.go:62] [tsf-tas/ctlog-createtree-job-mlckb-92rgz] pod update took 15.5572ms I0329 03:03:54.020923 3885 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-createtree-job-mlckb-92rgz] addLogicalPort annotation time took 15.582139ms I0329 03:03:54.022254 3885 pods.go:271] [tsf-tas/ctlog-createtree-job-mlckb-92rgz] addLogicalPort took 16.980987ms, libovsdb time 904.311µs I0329 03:03:54.166021 3885 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-6fbbdff645-xwkhb] creating logical port tsf-tas_trillian-logserver-6fbbdff645-xwkhb for pod on switch ip-10-0-2-173.ec2.internal I0329 03:03:54.166129 3885 kube.go:256] Updating pod tsf-tas/trillian-logserver-6fbbdff645-xwkhb I0329 03:03:54.182961 3885 pod.go:62] [tsf-tas/trillian-logserver-6fbbdff645-xwkhb] pod update took 16.863281ms I0329 03:03:54.182985 3885 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logserver-6fbbdff645-xwkhb] addLogicalPort annotation time took 16.894295ms I0329 03:03:54.184628 3885 pods.go:271] [tsf-tas/trillian-logserver-6fbbdff645-xwkhb] addLogicalPort took 18.625772ms, libovsdb time 1.198515ms I0329 03:03:54.298234 3885 cni.go:403] [tsf-tas/fulcio-server-576fd589f9-jbtbd 2b3d299df350162ee8842c369c39a83ac645e5f5675e4d3465217313bc43c0f2 network default NAD default] ADD starting CNI request [tsf-tas/fulcio-server-576fd589f9-jbtbd 2b3d299df350162ee8842c369c39a83ac645e5f5675e4d3465217313bc43c0f2 network default NAD default] I0329 03:03:54.307116 3885 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-576fd589f9-jbtbd, hostIfaceName: 2b3d299df350162, network: default, NAD default, SandboxID: "2b3d299df350162ee8842c369c39a83ac645e5f5675e4d3465217313bc43c0f2", PCI device ID: , UID: "a4c246db-b18b-44e0-96e5-96f79f5d958d", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0329 03:03:54.320721 3885 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-74947dcc6f-d9ncb] creating logical port tsf-tas_trillian-logsigner-74947dcc6f-d9ncb for pod on switch ip-10-0-2-173.ec2.internal I0329 03:03:54.320891 3885 kube.go:256] Updating pod tsf-tas/trillian-logsigner-74947dcc6f-d9ncb I0329 03:03:54.337037 3885 pod.go:62] [tsf-tas/trillian-logsigner-74947dcc6f-d9ncb] pod update took 16.176005ms I0329 03:03:54.337061 3885 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logsigner-74947dcc6f-d9ncb] addLogicalPort annotation time took 16.205672ms I0329 03:03:54.338691 3885 pods.go:271] [tsf-tas/trillian-logsigner-74947dcc6f-d9ncb] addLogicalPort took 17.984168ms, libovsdb time 1.091641ms I0329 03:03:54.358419 3885 cni.go:424] [tsf-tas/fulcio-server-576fd589f9-jbtbd 2b3d299df350162ee8842c369c39a83ac645e5f5675e4d3465217313bc43c0f2 network default NAD default] ADD finished CNI request [tsf-tas/fulcio-server-576fd589f9-jbtbd 2b3d299df350162ee8842c369c39a83ac645e5f5675e4d3465217313bc43c0f2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:27:6d:db:5c:67\",\"name\":\"2b3d299df350162\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d77fa57a-e1a6-42b5-a04c-1cb5590961bb\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:03:54.370068 3885 cni.go:403] [tsf-tas/ctlog-createtree-job-mlckb-92rgz 2fffff05a1fb1af5c77a763b8cfcf715c194915b8359b32f9afadab8614f5305 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-createtree-job-mlckb-92rgz 2fffff05a1fb1af5c77a763b8cfcf715c194915b8359b32f9afadab8614f5305 network default NAD default] I0329 03:03:54.378939 3885 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-mlckb-92rgz, hostIfaceName: 2fffff05a1fb1af, network: default, NAD default, SandboxID: "2fffff05a1fb1af5c77a763b8cfcf715c194915b8359b32f9afadab8614f5305", PCI device ID: , UID: "f89145f8-6275-433e-bd75-774ff09def72", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0329 03:03:54.636473 3885 cni.go:424] [tsf-tas/ctlog-createtree-job-mlckb-92rgz 2fffff05a1fb1af5c77a763b8cfcf715c194915b8359b32f9afadab8614f5305 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-mlckb-92rgz 2fffff05a1fb1af5c77a763b8cfcf715c194915b8359b32f9afadab8614f5305 network default NAD default], result "{\"interfaces\":[{\"mac\":\"de:b5:b6:0f:2f:0e\",\"name\":\"2fffff05a1fb1af\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5e4ab679-7d30-4619-9ef9-294092f32f76\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:03:55.135710 3885 cni.go:403] [tsf-tas/trillian-logserver-6fbbdff645-xwkhb a7a5881936f6b0614d577a4f65bc9d8ce6e803d7f1c559bc1b88713c034d80bc network default NAD default] ADD starting CNI request [tsf-tas/trillian-logserver-6fbbdff645-xwkhb a7a5881936f6b0614d577a4f65bc9d8ce6e803d7f1c559bc1b88713c034d80bc network default NAD default] I0329 03:03:55.144829 3885 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-6fbbdff645-xwkhb, hostIfaceName: a7a5881936f6b06, network: default, NAD default, SandboxID: "a7a5881936f6b0614d577a4f65bc9d8ce6e803d7f1c559bc1b88713c034d80bc", PCI device ID: , UID: "37ade0a9-4bd7-431e-9f99-7556feadb41c", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0329 03:03:55.192582 3885 cni.go:424] [tsf-tas/trillian-logserver-6fbbdff645-xwkhb a7a5881936f6b0614d577a4f65bc9d8ce6e803d7f1c559bc1b88713c034d80bc network default NAD default] ADD finished CNI request [tsf-tas/trillian-logserver-6fbbdff645-xwkhb a7a5881936f6b0614d577a4f65bc9d8ce6e803d7f1c559bc1b88713c034d80bc network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:04:84:7d:7b:6a\",\"name\":\"a7a5881936f6b06\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ef95e1c-fffd-4d87-896c-c80d63f5149e\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:03:55.293238 3885 cni.go:403] [tsf-tas/trillian-logsigner-74947dcc6f-d9ncb 7390613aaf2786d7edcc35a06787f2f4e38698eea9ea5c1ba09ec8a53ae2777f network default NAD default] ADD starting CNI request [tsf-tas/trillian-logsigner-74947dcc6f-d9ncb 7390613aaf2786d7edcc35a06787f2f4e38698eea9ea5c1ba09ec8a53ae2777f network default NAD default] I0329 03:03:55.302244 3885 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-74947dcc6f-d9ncb, hostIfaceName: 7390613aaf2786d, network: default, NAD default, SandboxID: "7390613aaf2786d7edcc35a06787f2f4e38698eea9ea5c1ba09ec8a53ae2777f", PCI device ID: , UID: "6ee5764e-1e7c-41be-b9c1-5fca7ec21dd8", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0329 03:03:55.559575 3885 cni.go:424] [tsf-tas/trillian-logsigner-74947dcc6f-d9ncb 7390613aaf2786d7edcc35a06787f2f4e38698eea9ea5c1ba09ec8a53ae2777f network default NAD default] ADD finished CNI request [tsf-tas/trillian-logsigner-74947dcc6f-d9ncb 7390613aaf2786d7edcc35a06787f2f4e38698eea9ea5c1ba09ec8a53ae2777f network default NAD default], result "{\"interfaces\":[{\"mac\":\"da:2a:41:26:7a:a1\",\"name\":\"7390613aaf2786d\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1ef86a81-13c2-4832-a21f-e53febb6fcbf\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 03:03:56.363594 3885 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0329 03:03:56.363623 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0329 03:03:56.398162 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0329 03:03:56.398250 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server I0329 03:04:03.053842 3885 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-4mqwg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:03.054401 3885 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-4mqwg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:04.081799 3885 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-4mqwg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:04.081830 3885 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-4mqwg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:04.427684 3885 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:04.428257 3885 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:04.908006 3885 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:04.908033 3885 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:04.916709 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0329 03:04:04.916737 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 I0329 03:04:06.558225 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=53 I0329 03:04:06.628726 3885 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-25s7b-vh7x4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:06.629307 3885 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-25s7b-vh7x4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0329 03:04:06.931479 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:04:06.931549 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0329 03:04:06.931520 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0329 03:04:07.440088 3885 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0329 03:04:07.649770 3885 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-25s7b-vh7x4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:07.649794 3885 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-25s7b-vh7x4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0329 03:04:10.453502 3885 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0329 03:04:10.481510 3885 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0329 03:04:15.994111 3885 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0329 03:04:28.148117 3885 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-crzfg] creating logical port openshift-pipelines_pipelines-console-plugin-79cb8f5447-crzfg for pod on switch ip-10-0-2-173.ec2.internal I0329 03:04:28.148214 3885 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-79cb8f5447-crzfg I0329 03:04:28.163321 3885 pod.go:62] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-crzfg] pod update took 15.130135ms I0329 03:04:28.163349 3885 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-crzfg] addLogicalPort annotation time took 15.162044ms I0329 03:04:28.164629 3885 pods.go:271] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-crzfg] addLogicalPort took 16.528853ms, libovsdb time 829.513µs I0329 03:04:28.508478 3885 cni.go:403] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-crzfg db83a29892f73702edc7385fc46d5e726d33f7cd94090a775cbe7e626794e148 network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-crzfg db83a29892f73702edc7385fc46d5e726d33f7cd94090a775cbe7e626794e148 network default NAD default] I0329 03:04:28.517104 3885 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-79cb8f5447-crzfg, hostIfaceName: db83a29892f7370, network: default, NAD default, SandboxID: "db83a29892f73702edc7385fc46d5e726d33f7cd94090a775cbe7e626794e148", PCI device ID: , UID: "81e97400-b58c-4148-9b60-be3358c09658", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0329 03:04:28.567799 3885 cni.go:424] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-crzfg db83a29892f73702edc7385fc46d5e726d33f7cd94090a775cbe7e626794e148 network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-crzfg db83a29892f73702edc7385fc46d5e726d33f7cd94090a775cbe7e626794e148 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:0a:81:8c:ce:ad\",\"name\":\"db83a29892f7370\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/56bcf8cd-621c-47c8-ae21-1dbc7e541a1d\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 03:04:33.546217 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver W0329 03:04:34.553236 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0329 03:04:38.658630 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0329 03:04:39.554985 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=46 I0329 03:04:41.241196 3885 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:41.241787 3885 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:41.262294 3885 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:41.262311 3885 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:41.269370 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0329 03:04:41.269395 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0329 03:04:49.295970 3885 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-jcwn7-q7g4l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:49.296548 3885 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-jcwn7-q7g4l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:50.178969 3885 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-jcwn7-q7g4l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:50.178993 3885 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-jcwn7-q7g4l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:50.299456 3885 obj_retry.go:464] Detected object tsf/patch-tekton-config-tzqlf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:50.300009 3885 obj_retry.go:464] Detected object tsf/patch-tekton-config-tzqlf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:51.033361 3885 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-7f5cf48bf6-sxg62] creating logical port tsf-tas_rekor-search-ui-7f5cf48bf6-sxg62 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:04:51.033456 3885 kube.go:256] Updating pod tsf-tas/rekor-search-ui-7f5cf48bf6-sxg62 I0329 03:04:51.048097 3885 pod.go:62] [tsf-tas/rekor-search-ui-7f5cf48bf6-sxg62] pod update took 14.661764ms I0329 03:04:51.048123 3885 base_network_controller_pods.go:937] [default/tsf-tas/rekor-search-ui-7f5cf48bf6-sxg62] addLogicalPort annotation time took 14.697357ms I0329 03:04:51.049503 3885 pods.go:271] [tsf-tas/rekor-search-ui-7f5cf48bf6-sxg62] addLogicalPort took 16.15734ms, libovsdb time 831.475µs I0329 03:04:51.322278 3885 obj_retry.go:458] Detected object tsf/patch-tekton-config-tzqlf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:51.322299 3885 obj_retry.go:458] Detected object tsf/patch-tekton-config-tzqlf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:51.421863 3885 cni.go:403] [tsf-tas/rekor-search-ui-7f5cf48bf6-sxg62 3b68679716dd3895fdc80eb84b66b6b706ce04fb5c7230c7e7da674171776f76 network default NAD default] ADD starting CNI request [tsf-tas/rekor-search-ui-7f5cf48bf6-sxg62 3b68679716dd3895fdc80eb84b66b6b706ce04fb5c7230c7e7da674171776f76 network default NAD default] I0329 03:04:51.430322 3885 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-7f5cf48bf6-sxg62, hostIfaceName: 3b68679716dd389, network: default, NAD default, SandboxID: "3b68679716dd3895fdc80eb84b66b6b706ce04fb5c7230c7e7da674171776f76", PCI device ID: , UID: "efd9c90f-c167-4618-8fa2-bf37191e43ce", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0329 03:04:51.684803 3885 cni.go:424] [tsf-tas/rekor-search-ui-7f5cf48bf6-sxg62 3b68679716dd3895fdc80eb84b66b6b706ce04fb5c7230c7e7da674171776f76 network default NAD default] ADD finished CNI request [tsf-tas/rekor-search-ui-7f5cf48bf6-sxg62 3b68679716dd3895fdc80eb84b66b6b706ce04fb5c7230c7e7da674171776f76 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ce:10:c1:3a:be:50\",\"name\":\"3b68679716dd389\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e2d7b03f-86ea-435f-8110-dedf6a84b1b8\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 03:04:54.184240 3885 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0329 03:04:56.741964 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0329 03:04:57.755693 3885 cni.go:403] [tsf-tas/ctlog-createtree-job-mlckb-92rgz 2fffff05a1fb1af5c77a763b8cfcf715c194915b8359b32f9afadab8614f5305 network default NAD default] DEL starting CNI request [tsf-tas/ctlog-createtree-job-mlckb-92rgz 2fffff05a1fb1af5c77a763b8cfcf715c194915b8359b32f9afadab8614f5305 network default NAD default] I0329 03:04:57.840832 3885 cni.go:424] [tsf-tas/ctlog-createtree-job-mlckb-92rgz 2fffff05a1fb1af5c77a763b8cfcf715c194915b8359b32f9afadab8614f5305 network default NAD default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-mlckb-92rgz 2fffff05a1fb1af5c77a763b8cfcf715c194915b8359b32f9afadab8614f5305 network default NAD default], result "{}", err I0329 03:04:57.892742 3885 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-mlckb-92rgz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:57.892775 3885 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-mlckb-92rgz I0329 03:04:57.894147 3885 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-mlckb-92rgz, ips: 10.128.0.53 I0329 03:04:57.894175 3885 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-mlckb-92rgz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:58.792012 3885 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-mlckb-92rgz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:58.792038 3885 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-mlckb-92rgz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:58.806484 3885 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-j5v7l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:58.806972 3885 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-j5v7l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:04:59.685541 3885 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-j5v7l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:59.685567 3885 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-j5v7l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:59.736285 3885 obj_retry.go:458] Detected object tsf/patch-tekton-config-tzqlf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:59.736309 3885 obj_retry.go:458] Detected object tsf/patch-tekton-config-tzqlf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:04:59.743852 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-tzqlf I0329 03:04:59.743876 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-tzqlf I0329 03:04:59.858851 3885 base_network_controller_pods.go:487] [default/tsf/test-tsf-pipelines] creating logical port tsf_test-tsf-pipelines for pod on switch ip-10-0-2-173.ec2.internal I0329 03:04:59.858946 3885 kube.go:256] Updating pod tsf/test-tsf-pipelines I0329 03:04:59.873654 3885 pod.go:62] [tsf/test-tsf-pipelines] pod update took 14.730098ms I0329 03:04:59.873687 3885 base_network_controller_pods.go:937] [default/tsf/test-tsf-pipelines] addLogicalPort annotation time took 14.766896ms I0329 03:04:59.875056 3885 pods.go:271] [tsf/test-tsf-pipelines] addLogicalPort took 16.226956ms, libovsdb time 930.413µs I0329 03:05:00.221341 3885 cni.go:403] [tsf/test-tsf-pipelines 858b11da77333bcf1f4b18a1d9150a95eda2c7471ffa1f0668f6bb227e9e1448 network default NAD default] ADD starting CNI request [tsf/test-tsf-pipelines 858b11da77333bcf1f4b18a1d9150a95eda2c7471ffa1f0668f6bb227e9e1448 network default NAD default] I0329 03:05:00.230329 3885 helper_linux.go:450] ConfigureOVS: namespace: tsf, podName: test-tsf-pipelines, hostIfaceName: 858b11da77333bc, network: default, NAD default, SandboxID: "858b11da77333bcf1f4b18a1d9150a95eda2c7471ffa1f0668f6bb227e9e1448", PCI device ID: , UID: "8343779d-10c2-4c64-a826-fba9261389e7", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0329 03:05:00.278597 3885 cni.go:424] [tsf/test-tsf-pipelines 858b11da77333bcf1f4b18a1d9150a95eda2c7471ffa1f0668f6bb227e9e1448 network default NAD default] ADD finished CNI request [tsf/test-tsf-pipelines 858b11da77333bcf1f4b18a1d9150a95eda2c7471ffa1f0668f6bb227e9e1448 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:66:f3:b8:c5:99\",\"name\":\"858b11da77333bc\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0f65bc21-1ac9-407d-a4d1-42b0fed3fc8c\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 03:05:03.696730 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0329 03:05:05.240438 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server W0329 03:05:08.855436 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0329 03:05:10.846494 3885 cni.go:403] [tsf/test-tsf-pipelines 858b11da77333bcf1f4b18a1d9150a95eda2c7471ffa1f0668f6bb227e9e1448 network default NAD default] DEL starting CNI request [tsf/test-tsf-pipelines 858b11da77333bcf1f4b18a1d9150a95eda2c7471ffa1f0668f6bb227e9e1448 network default NAD default] I0329 03:05:10.928479 3885 cni.go:424] [tsf/test-tsf-pipelines 858b11da77333bcf1f4b18a1d9150a95eda2c7471ffa1f0668f6bb227e9e1448 network default NAD default] DEL finished CNI request [tsf/test-tsf-pipelines 858b11da77333bcf1f4b18a1d9150a95eda2c7471ffa1f0668f6bb227e9e1448 network default NAD default], result "{}", err I0329 03:05:10.981325 3885 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:10.981360 3885 pods.go:176] Deleting pod: tsf/test-tsf-pipelines I0329 03:05:10.982720 3885 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-pipelines, ips: 10.128.0.58 I0329 03:05:10.982753 3885 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:11.000413 3885 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:11.000434 3885 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:11.008995 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0329 03:05:11.009020 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0329 03:05:11.562537 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=99 I0329 03:05:15.434854 3885 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:15.435517 3885 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:15.456454 3885 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:15.456497 3885 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:15.464866 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0329 03:05:15.464894 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0329 03:05:22.944232 3885 obj_retry.go:464] Detected object tsf-tpa/create-db-w97nj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:22.944818 3885 obj_retry.go:464] Detected object tsf-tpa/create-db-w97nj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:23.823809 3885 obj_retry.go:458] Detected object tsf-tpa/create-db-w97nj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:23.823833 3885 obj_retry.go:458] Detected object tsf-tpa/create-db-w97nj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:23.930186 3885 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-c46dv] creating logical port tsf-tpa_migrate-db-c46dv for pod on switch ip-10-0-2-173.ec2.internal I0329 03:05:23.930309 3885 kube.go:256] Updating pod tsf-tpa/migrate-db-c46dv I0329 03:05:23.942797 3885 pod.go:62] [tsf-tpa/migrate-db-c46dv] pod update took 12.511034ms I0329 03:05:23.942822 3885 base_network_controller_pods.go:937] [default/tsf-tpa/migrate-db-c46dv] addLogicalPort annotation time took 12.541625ms I0329 03:05:23.944156 3885 pods.go:271] [tsf-tpa/migrate-db-c46dv] addLogicalPort took 13.990925ms, libovsdb time 866.642µs I0329 03:05:24.293093 3885 cni.go:403] [tsf-tpa/migrate-db-c46dv f56910129f2ce9a1c748ee3541d19bfcb90eb2600d6c4e3c14abaa9dbc787a8c network default NAD default] ADD starting CNI request [tsf-tpa/migrate-db-c46dv f56910129f2ce9a1c748ee3541d19bfcb90eb2600d6c4e3c14abaa9dbc787a8c network default NAD default] I0329 03:05:24.302699 3885 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-c46dv, hostIfaceName: f56910129f2ce9a, network: default, NAD default, SandboxID: "f56910129f2ce9a1c748ee3541d19bfcb90eb2600d6c4e3c14abaa9dbc787a8c", PCI device ID: , UID: "db5551ed-2fd3-4801-9502-cecaa7aa0518", MAC: 0a:58:0a:80:00:3b, IPs: [10.128.0.59/23] I0329 03:05:24.351824 3885 cni.go:424] [tsf-tpa/migrate-db-c46dv f56910129f2ce9a1c748ee3541d19bfcb90eb2600d6c4e3c14abaa9dbc787a8c network default NAD default] ADD finished CNI request [tsf-tpa/migrate-db-c46dv f56910129f2ce9a1c748ee3541d19bfcb90eb2600d6c4e3c14abaa9dbc787a8c network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:73:70:9b:59:1e\",\"name\":\"f56910129f2ce9a\"},{\"mac\":\"0a:58:0a:80:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c5488522-3557-44bc-9052-2f8e4854da65\"}],\"ips\":[{\"address\":\"10.128.0.59/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:05:28.510553 3885 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-x9m9g-696dh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:28.511089 3885 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-x9m9g-696dh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:29.535057 3885 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-x9m9g-696dh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:29.535081 3885 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-x9m9g-696dh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:29.970934 3885 cni.go:403] [tsf-tpa/migrate-db-c46dv f56910129f2ce9a1c748ee3541d19bfcb90eb2600d6c4e3c14abaa9dbc787a8c network default NAD default] DEL starting CNI request [tsf-tpa/migrate-db-c46dv f56910129f2ce9a1c748ee3541d19bfcb90eb2600d6c4e3c14abaa9dbc787a8c network default NAD default] I0329 03:05:30.043497 3885 cni.go:424] [tsf-tpa/migrate-db-c46dv f56910129f2ce9a1c748ee3541d19bfcb90eb2600d6c4e3c14abaa9dbc787a8c network default NAD default] DEL finished CNI request [tsf-tpa/migrate-db-c46dv f56910129f2ce9a1c748ee3541d19bfcb90eb2600d6c4e3c14abaa9dbc787a8c network default NAD default], result "{}", err I0329 03:05:30.096201 3885 obj_retry.go:464] Detected object tsf-tpa/migrate-db-c46dv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:30.096232 3885 pods.go:176] Deleting pod: tsf-tpa/migrate-db-c46dv I0329 03:05:30.097621 3885 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-c46dv, ips: 10.128.0.59 I0329 03:05:30.097657 3885 obj_retry.go:464] Detected object tsf-tpa/migrate-db-c46dv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:30.988005 3885 obj_retry.go:458] Detected object tsf-tpa/migrate-db-c46dv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:30.988035 3885 obj_retry.go:458] Detected object tsf-tpa/migrate-db-c46dv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:31.662952 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0329 03:05:35.558372 3885 obj_retry.go:464] Detected object tsf-tpa/create-importers-v6xfb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:35.559007 3885 obj_retry.go:464] Detected object tsf-tpa/create-importers-v6xfb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:05:36.582195 3885 obj_retry.go:458] Detected object tsf-tpa/create-importers-v6xfb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:36.582225 3885 obj_retry.go:458] Detected object tsf-tpa/create-importers-v6xfb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:05:37.685560 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 W0329 03:05:43.930818 3885 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0329 03:05:46.479191 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0329 03:06:09.462907 3885 namespace.go:142] [openshift-pipelines] updating namespace I0329 03:06:11.743372 3885 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:11.743935 3885 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:11.785258 3885 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:11.785279 3885 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:11.793673 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0329 03:06:11.793693 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0329 03:06:12.517635 3885 namespace.go:98] [integration-service] adding namespace I0329 03:06:12.518647 3885 namespace.go:102] [integration-service] adding namespace took 926.159µs I0329 03:06:12.522907 3885 namespace.go:98] [image-controller] adding namespace I0329 03:06:12.523635 3885 namespace.go:102] [image-controller] adding namespace took 710.594µs I0329 03:06:12.526550 3885 namespace.go:98] [build-service] adding namespace I0329 03:06:12.527224 3885 namespace.go:102] [build-service] adding namespace took 660.266µs I0329 03:06:12.564616 3885 namespace.go:142] [integration-service] updating namespace I0329 03:06:12.587958 3885 namespace.go:142] [integration-service] updating namespace I0329 03:06:12.593286 3885 namespace.go:142] [image-controller] updating namespace I0329 03:06:12.620600 3885 namespace.go:142] [image-controller] updating namespace I0329 03:06:12.631589 3885 namespace.go:142] [build-service] updating namespace I0329 03:06:12.651776 3885 namespace.go:142] [build-service] updating namespace I0329 03:06:12.715846 3885 namespace.go:142] [build-service] updating namespace I0329 03:06:12.752892 3885 namespace.go:142] [integration-service] updating namespace I0329 03:06:12.801509 3885 namespace.go:98] [release-service] adding namespace I0329 03:06:12.802274 3885 namespace.go:102] [release-service] adding namespace took 747.593µs I0329 03:06:12.842518 3885 base_network_controller_pods.go:487] [default/konflux-ui/tsf-konflux-public-key-2z7x9] creating logical port konflux-ui_tsf-konflux-public-key-2z7x9 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:06:12.842648 3885 kube.go:256] Updating pod konflux-ui/tsf-konflux-public-key-2z7x9 I0329 03:06:12.858106 3885 namespace.go:142] [release-service] updating namespace I0329 03:06:12.876763 3885 namespace.go:142] [release-service] updating namespace I0329 03:06:12.888519 3885 pod.go:62] [konflux-ui/tsf-konflux-public-key-2z7x9] pod update took 45.892283ms I0329 03:06:12.888537 3885 base_network_controller_pods.go:937] [default/konflux-ui/tsf-konflux-public-key-2z7x9] addLogicalPort annotation time took 45.917378ms I0329 03:06:12.889759 3885 pods.go:271] [konflux-ui/tsf-konflux-public-key-2z7x9] addLogicalPort took 47.254759ms, libovsdb time 814.13µs I0329 03:06:12.897678 3885 namespace.go:142] [konflux-ui] updating namespace I0329 03:06:12.974652 3885 namespace.go:98] [konflux-info] adding namespace I0329 03:06:12.975369 3885 namespace.go:102] [konflux-info] adding namespace took 696.765µs I0329 03:06:13.032549 3885 namespace.go:98] [namespace-lister] adding namespace I0329 03:06:13.033303 3885 namespace.go:102] [namespace-lister] adding namespace took 735.241µs I0329 03:06:13.071930 3885 namespace.go:142] [konflux-info] updating namespace I0329 03:06:13.123523 3885 namespace.go:98] [enterprise-contract-service] adding namespace I0329 03:06:13.123653 3885 namespace.go:142] [image-controller] updating namespace I0329 03:06:13.124249 3885 namespace.go:102] [enterprise-contract-service] adding namespace took 708.971µs I0329 03:06:13.154259 3885 namespace.go:142] [konflux-info] updating namespace I0329 03:06:13.195778 3885 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:13.202553 3885 cni.go:403] [konflux-ui/tsf-konflux-public-key-2z7x9 8f804a69a25d8d8cdd60b0712178c6e6133a8222d378b67deed22ceffd0f248d network default NAD default] ADD starting CNI request [konflux-ui/tsf-konflux-public-key-2z7x9 8f804a69a25d8d8cdd60b0712178c6e6133a8222d378b67deed22ceffd0f248d network default NAD default] I0329 03:06:13.211301 3885 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: tsf-konflux-public-key-2z7x9, hostIfaceName: 8f804a69a25d8d8, network: default, NAD default, SandboxID: "8f804a69a25d8d8cdd60b0712178c6e6133a8222d378b67deed22ceffd0f248d", PCI device ID: , UID: "0b9c2fc1-1e09-42f9-b953-482b0d73a645", MAC: 0a:58:0a:80:00:3c, IPs: [10.128.0.60/23] I0329 03:06:13.253798 3885 namespace.go:142] [enterprise-contract-service] updating namespace I0329 03:06:13.253925 3885 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:13.260086 3885 cni.go:424] [konflux-ui/tsf-konflux-public-key-2z7x9 8f804a69a25d8d8cdd60b0712178c6e6133a8222d378b67deed22ceffd0f248d network default NAD default] ADD finished CNI request [konflux-ui/tsf-konflux-public-key-2z7x9 8f804a69a25d8d8cdd60b0712178c6e6133a8222d378b67deed22ceffd0f248d network default NAD default], result "{\"interfaces\":[{\"mac\":\"72:f3:17:7e:58:a2\",\"name\":\"8f804a69a25d8d8\"},{\"mac\":\"0a:58:0a:80:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ab1baa11-bd70-450e-a0c0-ce691ec5126a\"}],\"ips\":[{\"address\":\"10.128.0.60/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:06:13.361863 3885 namespace.go:142] [enterprise-contract-service] updating namespace I0329 03:06:13.365952 3885 namespace.go:98] [default-tenant] adding namespace I0329 03:06:13.366744 3885 namespace.go:102] [default-tenant] adding namespace took 773.914µs I0329 03:06:13.449830 3885 namespace.go:142] [default-tenant] updating namespace I0329 03:06:13.454602 3885 namespace.go:98] [segment-bridge] adding namespace I0329 03:06:13.455351 3885 namespace.go:102] [segment-bridge] adding namespace took 728.75µs I0329 03:06:13.579291 3885 namespace.go:142] [segment-bridge] updating namespace I0329 03:06:13.590760 3885 namespace.go:142] [default-tenant] updating namespace I0329 03:06:13.725751 3885 namespace.go:142] [segment-bridge] updating namespace I0329 03:06:14.236685 3885 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0329 03:06:14.237337 3885 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0329 03:06:14.237358 3885 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0329 03:06:14.240060 3885 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0329 03:06:14.315585 3885 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0329 03:06:14.315609 3885 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0329 03:06:14.316253 3885 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0329 03:06:14.358649 3885 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-67655c4674-pb8xm] creating logical port namespace-lister_namespace-lister-67655c4674-pb8xm for pod on switch ip-10-0-2-173.ec2.internal I0329 03:06:14.358730 3885 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-pb8xm I0329 03:06:14.411437 3885 pod.go:62] [namespace-lister/namespace-lister-67655c4674-pb8xm] pod update took 52.731394ms I0329 03:06:14.411459 3885 base_network_controller_pods.go:937] [default/namespace-lister/namespace-lister-67655c4674-pb8xm] addLogicalPort annotation time took 52.757972ms I0329 03:06:14.412802 3885 pods.go:271] [namespace-lister/namespace-lister-67655c4674-pb8xm] addLogicalPort took 54.162291ms, libovsdb time 905.077µs I0329 03:06:14.634615 3885 namespace.go:142] [build-service] updating namespace I0329 03:06:14.940700 3885 base_network_controller_pods.go:487] [default/konflux-ui/dex-5d9f556c67-cm7pp] creating logical port konflux-ui_dex-5d9f556c67-cm7pp for pod on switch ip-10-0-2-173.ec2.internal I0329 03:06:14.940782 3885 kube.go:256] Updating pod konflux-ui/dex-5d9f556c67-cm7pp I0329 03:06:14.961112 3885 pod.go:62] [konflux-ui/dex-5d9f556c67-cm7pp] pod update took 20.354325ms I0329 03:06:14.961135 3885 base_network_controller_pods.go:937] [default/konflux-ui/dex-5d9f556c67-cm7pp] addLogicalPort annotation time took 20.380497ms I0329 03:06:14.962356 3885 pods.go:271] [konflux-ui/dex-5d9f556c67-cm7pp] addLogicalPort took 21.668983ms, libovsdb time 793.667µs I0329 03:06:15.051169 3885 base_network_controller_pods.go:487] [default/konflux-ui/proxy-5bbc66485b-k45f5] creating logical port konflux-ui_proxy-5bbc66485b-k45f5 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:06:15.051254 3885 kube.go:256] Updating pod konflux-ui/proxy-5bbc66485b-k45f5 I0329 03:06:15.083997 3885 pod.go:62] [konflux-ui/proxy-5bbc66485b-k45f5] pod update took 32.772931ms I0329 03:06:15.084016 3885 base_network_controller_pods.go:937] [default/konflux-ui/proxy-5bbc66485b-k45f5] addLogicalPort annotation time took 32.797542ms I0329 03:06:15.085280 3885 pods.go:271] [konflux-ui/proxy-5bbc66485b-k45f5] addLogicalPort took 34.123901ms, libovsdb time 856.427µs I0329 03:06:15.123600 3885 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz] creating logical port image-controller_image-controller-controller-manager-5f8cd64d55-t5tlz for pod on switch ip-10-0-2-173.ec2.internal I0329 03:06:15.123677 3885 kube.go:256] Updating pod image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz I0329 03:06:15.125828 3885 namespace.go:142] [image-controller] updating namespace I0329 03:06:15.139129 3885 pod.go:62] [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz] pod update took 15.470742ms I0329 03:06:15.139151 3885 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz] addLogicalPort annotation time took 15.494159ms I0329 03:06:15.140432 3885 pods.go:271] [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz] addLogicalPort took 16.847388ms, libovsdb time 756.684µs I0329 03:06:15.245688 3885 cni.go:403] [konflux-ui/tsf-konflux-public-key-2z7x9 8f804a69a25d8d8cdd60b0712178c6e6133a8222d378b67deed22ceffd0f248d network default NAD default] DEL starting CNI request [konflux-ui/tsf-konflux-public-key-2z7x9 8f804a69a25d8d8cdd60b0712178c6e6133a8222d378b67deed22ceffd0f248d network default NAD default] I0329 03:06:15.323016 3885 cni.go:403] [namespace-lister/namespace-lister-67655c4674-pb8xm 1d98512bff534519ba85784a75cbacf8edffc488b42b5d8eb79235776e031213 network default NAD default] ADD starting CNI request [namespace-lister/namespace-lister-67655c4674-pb8xm 1d98512bff534519ba85784a75cbacf8edffc488b42b5d8eb79235776e031213 network default NAD default] I0329 03:06:15.332502 3885 helper_linux.go:450] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-67655c4674-pb8xm, hostIfaceName: 1d98512bff53451, network: default, NAD default, SandboxID: "1d98512bff534519ba85784a75cbacf8edffc488b42b5d8eb79235776e031213", PCI device ID: , UID: "934ff63e-5a3b-45c8-bc5e-cff7f59d673f", MAC: 0a:58:0a:80:00:3d, IPs: [10.128.0.61/23] I0329 03:06:15.333225 3885 cni.go:424] [konflux-ui/tsf-konflux-public-key-2z7x9 8f804a69a25d8d8cdd60b0712178c6e6133a8222d378b67deed22ceffd0f248d network default NAD default] DEL finished CNI request [konflux-ui/tsf-konflux-public-key-2z7x9 8f804a69a25d8d8cdd60b0712178c6e6133a8222d378b67deed22ceffd0f248d network default NAD default], result "{}", err I0329 03:06:15.382898 3885 cni.go:424] [namespace-lister/namespace-lister-67655c4674-pb8xm 1d98512bff534519ba85784a75cbacf8edffc488b42b5d8eb79235776e031213 network default NAD default] ADD finished CNI request [namespace-lister/namespace-lister-67655c4674-pb8xm 1d98512bff534519ba85784a75cbacf8edffc488b42b5d8eb79235776e031213 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:33:fc:b5:cf:29\",\"name\":\"1d98512bff53451\"},{\"mac\":\"0a:58:0a:80:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0ccc4857-0401-47dc-855b-7cbe49d08e6e\"}],\"ips\":[{\"address\":\"10.128.0.61/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:06:15.395321 3885 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-759977d69f-rbtgb] creating logical port integration-service_integration-service-controller-manager-759977d69f-rbtgb for pod on switch ip-10-0-2-173.ec2.internal I0329 03:06:15.395524 3885 kube.go:256] Updating pod integration-service/integration-service-controller-manager-759977d69f-rbtgb I0329 03:06:15.413946 3885 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-2z7x9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:15.413986 3885 pods.go:176] Deleting pod: konflux-ui/tsf-konflux-public-key-2z7x9 I0329 03:06:15.415416 3885 pods.go:236] Attempting to release IPs for pod: konflux-ui/tsf-konflux-public-key-2z7x9, ips: 10.128.0.60 I0329 03:06:15.415451 3885 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-2z7x9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:15.425057 3885 namespace.go:142] [integration-service] updating namespace I0329 03:06:15.431539 3885 pod.go:62] [integration-service/integration-service-controller-manager-759977d69f-rbtgb] pod update took 36.090453ms I0329 03:06:15.431560 3885 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-759977d69f-rbtgb] addLogicalPort annotation time took 36.116018ms I0329 03:06:15.432819 3885 pods.go:271] [integration-service/integration-service-controller-manager-759977d69f-rbtgb] addLogicalPort took 37.526354ms, libovsdb time 794.484µs I0329 03:06:15.498343 3885 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz 9d6ccc148a886f469d1ea8066d7161710793271dbe6ce860b6c3d6020fb0af30 network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz 9d6ccc148a886f469d1ea8066d7161710793271dbe6ce860b6c3d6020fb0af30 network default NAD default] I0329 03:06:15.507438 3885 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-5f8cd64d55-t5tlz, hostIfaceName: 9d6ccc148a886f4, network: default, NAD default, SandboxID: "9d6ccc148a886f469d1ea8066d7161710793271dbe6ce860b6c3d6020fb0af30", PCI device ID: , UID: "9dd6d2bb-f776-4871-a142-fabe6d8f37f7", MAC: 0a:58:0a:80:00:40, IPs: [10.128.0.64/23] I0329 03:06:15.555537 3885 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz 9d6ccc148a886f469d1ea8066d7161710793271dbe6ce860b6c3d6020fb0af30 network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz 9d6ccc148a886f469d1ea8066d7161710793271dbe6ce860b6c3d6020fb0af30 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:44:89:bc:75:88\",\"name\":\"9d6ccc148a886f4\"},{\"mac\":\"0a:58:0a:80:00:40\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9c7d514c-e84c-4655-87a6-0c6e770d638f\"}],\"ips\":[{\"address\":\"10.128.0.64/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:06:15.579377 3885 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7df75dd4df-5hzzb] creating logical port build-service_build-service-controller-manager-7df75dd4df-5hzzb for pod on switch ip-10-0-2-173.ec2.internal I0329 03:06:15.579544 3885 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-5hzzb I0329 03:06:15.608273 3885 pod.go:62] [build-service/build-service-controller-manager-7df75dd4df-5hzzb] pod update took 28.761421ms I0329 03:06:15.608297 3885 base_network_controller_pods.go:937] [default/build-service/build-service-controller-manager-7df75dd4df-5hzzb] addLogicalPort annotation time took 28.790825ms I0329 03:06:15.609629 3885 pods.go:271] [build-service/build-service-controller-manager-7df75dd4df-5hzzb] addLogicalPort took 30.270444ms, libovsdb time 845.009µs I0329 03:06:15.633718 3885 namespace.go:142] [konflux-info] updating namespace I0329 03:06:15.837411 3885 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:15.952642 3885 cni.go:403] [build-service/build-service-controller-manager-7df75dd4df-5hzzb 17206b86fdc48850f173da702f2349c4e7b7dbabe9a6547dd77d358e1eb1c071 network default NAD default] ADD starting CNI request [build-service/build-service-controller-manager-7df75dd4df-5hzzb 17206b86fdc48850f173da702f2349c4e7b7dbabe9a6547dd77d358e1eb1c071 network default NAD default] I0329 03:06:15.960913 3885 helper_linux.go:450] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7df75dd4df-5hzzb, hostIfaceName: 17206b86fdc4885, network: default, NAD default, SandboxID: "17206b86fdc48850f173da702f2349c4e7b7dbabe9a6547dd77d358e1eb1c071", PCI device ID: , UID: "aaf53d3e-f532-42ae-b525-128ab1a5f612", MAC: 0a:58:0a:80:00:42, IPs: [10.128.0.66/23] I0329 03:06:16.008876 3885 cni.go:424] [build-service/build-service-controller-manager-7df75dd4df-5hzzb 17206b86fdc48850f173da702f2349c4e7b7dbabe9a6547dd77d358e1eb1c071 network default NAD default] ADD finished CNI request [build-service/build-service-controller-manager-7df75dd4df-5hzzb 17206b86fdc48850f173da702f2349c4e7b7dbabe9a6547dd77d358e1eb1c071 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f2:d4:94:f4:32:c2\",\"name\":\"17206b86fdc4885\"},{\"mac\":\"0a:58:0a:80:00:42\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c285ece9-ec4f-4c81-b185-43be4805e25c\"}],\"ips\":[{\"address\":\"10.128.0.66/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:06:16.010768 3885 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-854f775895-zw2g8] creating logical port release-service_release-service-controller-manager-854f775895-zw2g8 for pod on switch ip-10-0-2-173.ec2.internal I0329 03:06:16.010914 3885 kube.go:256] Updating pod release-service/release-service-controller-manager-854f775895-zw2g8 I0329 03:06:16.027014 3885 namespace.go:142] [release-service] updating namespace I0329 03:06:16.042845 3885 pod.go:62] [release-service/release-service-controller-manager-854f775895-zw2g8] pod update took 31.960126ms I0329 03:06:16.042873 3885 base_network_controller_pods.go:937] [default/release-service/release-service-controller-manager-854f775895-zw2g8] addLogicalPort annotation time took 31.992342ms I0329 03:06:16.044160 3885 pods.go:271] [release-service/release-service-controller-manager-854f775895-zw2g8] addLogicalPort took 33.41152ms, libovsdb time 864.461µs I0329 03:06:16.156194 3885 namespace.go:142] [build-service] updating namespace I0329 03:06:16.172952 3885 namespace.go:142] [image-controller] updating namespace I0329 03:06:16.185110 3885 namespace.go:142] [integration-service] updating namespace I0329 03:06:16.203895 3885 namespace.go:142] [konflux-info] updating namespace I0329 03:06:16.225642 3885 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:16.237162 3885 namespace.go:142] [release-service] updating namespace I0329 03:06:16.283837 3885 namespace.go:142] [build-service] updating namespace I0329 03:06:16.333841 3885 namespace.go:142] [image-controller] updating namespace I0329 03:06:16.396631 3885 namespace.go:142] [integration-service] updating namespace I0329 03:06:16.427784 3885 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-2z7x9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:16.427809 3885 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-2z7x9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:16.502255 3885 namespace.go:142] [konflux-info] updating namespace I0329 03:06:16.605401 3885 namespace.go:142] [namespace-lister] updating namespace I0329 03:06:16.697168 3885 namespace.go:142] [release-service] updating namespace W0329 03:06:17.289378 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0329 03:06:17.912739 3885 namespace.go:142] [default-tenant] updating namespace I0329 03:06:18.113215 3885 namespace.go:142] [enterprise-contract-service] updating namespace W0329 03:06:18.289790 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0329 03:06:18.321906 3885 namespace.go:142] [segment-bridge] updating namespace I0329 03:06:18.439242 3885 namespace.go:142] [default-tenant] updating namespace I0329 03:06:18.457411 3885 namespace.go:142] [enterprise-contract-service] updating namespace I0329 03:06:18.474635 3885 namespace.go:142] [segment-bridge] updating namespace I0329 03:06:18.519617 3885 namespace.go:142] [default-tenant] updating namespace I0329 03:06:18.601547 3885 namespace.go:142] [enterprise-contract-service] updating namespace I0329 03:06:18.702866 3885 namespace.go:142] [segment-bridge] updating namespace W0329 03:06:20.287587 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0329 03:06:20.314495 3885 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0329 03:06:22.812141 3885 cni.go:403] [konflux-ui/dex-5d9f556c67-cm7pp 18737ce8ecc2b7b130c035f7b1ff310f4951a6711d5dcbf20260d516ff960775 network default NAD default] ADD starting CNI request [konflux-ui/dex-5d9f556c67-cm7pp 18737ce8ecc2b7b130c035f7b1ff310f4951a6711d5dcbf20260d516ff960775 network default NAD default] I0329 03:06:22.820737 3885 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: dex-5d9f556c67-cm7pp, hostIfaceName: 18737ce8ecc2b7b, network: default, NAD default, SandboxID: "18737ce8ecc2b7b130c035f7b1ff310f4951a6711d5dcbf20260d516ff960775", PCI device ID: , UID: "6fde9d9e-dfa8-484d-a066-53f2bf56a76d", MAC: 0a:58:0a:80:00:3e, IPs: [10.128.0.62/23] I0329 03:06:22.870632 3885 cni.go:424] [konflux-ui/dex-5d9f556c67-cm7pp 18737ce8ecc2b7b130c035f7b1ff310f4951a6711d5dcbf20260d516ff960775 network default NAD default] ADD finished CNI request [konflux-ui/dex-5d9f556c67-cm7pp 18737ce8ecc2b7b130c035f7b1ff310f4951a6711d5dcbf20260d516ff960775 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:42:76:6e:da:cc\",\"name\":\"18737ce8ecc2b7b\"},{\"mac\":\"0a:58:0a:80:00:3e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ed5cf20b-6e1b-4d03-9973-fc2b827a2c2c\"}],\"ips\":[{\"address\":\"10.128.0.62/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:06:22.926352 3885 cni.go:403] [konflux-ui/proxy-5bbc66485b-k45f5 3b93e0284ebcadd47115a0cf7a82e6d6f6debd01e22896fe83192e0455511edb network default NAD default] ADD starting CNI request [konflux-ui/proxy-5bbc66485b-k45f5 3b93e0284ebcadd47115a0cf7a82e6d6f6debd01e22896fe83192e0455511edb network default NAD default] I0329 03:06:22.935555 3885 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-5bbc66485b-k45f5, hostIfaceName: 3b93e0284ebcadd, network: default, NAD default, SandboxID: "3b93e0284ebcadd47115a0cf7a82e6d6f6debd01e22896fe83192e0455511edb", PCI device ID: , UID: "3f06c022-67d1-46c4-b994-f4519bfad494", MAC: 0a:58:0a:80:00:3f, IPs: [10.128.0.63/23] I0329 03:06:22.983109 3885 cni.go:424] [konflux-ui/proxy-5bbc66485b-k45f5 3b93e0284ebcadd47115a0cf7a82e6d6f6debd01e22896fe83192e0455511edb network default NAD default] ADD finished CNI request [konflux-ui/proxy-5bbc66485b-k45f5 3b93e0284ebcadd47115a0cf7a82e6d6f6debd01e22896fe83192e0455511edb network default NAD default], result "{\"interfaces\":[{\"mac\":\"0e:42:55:69:e9:d9\",\"name\":\"3b93e0284ebcadd\"},{\"mac\":\"0a:58:0a:80:00:3f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bfb4d6cb-a16d-4fa2-bc9b-f3c3e1a7bc28\"}],\"ips\":[{\"address\":\"10.128.0.63/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:06:23.268267 3885 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-rbtgb b2b06cb274a48965bc233adcac715996e29573ab1385a836821fca8b9a6647bc network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-759977d69f-rbtgb b2b06cb274a48965bc233adcac715996e29573ab1385a836821fca8b9a6647bc network default NAD default] I0329 03:06:23.277065 3885 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-759977d69f-rbtgb, hostIfaceName: b2b06cb274a4896, network: default, NAD default, SandboxID: "b2b06cb274a48965bc233adcac715996e29573ab1385a836821fca8b9a6647bc", PCI device ID: , UID: "4ef85116-655e-41dd-87d8-63af007d6c8f", MAC: 0a:58:0a:80:00:41, IPs: [10.128.0.65/23] I0329 03:06:23.324368 3885 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-rbtgb b2b06cb274a48965bc233adcac715996e29573ab1385a836821fca8b9a6647bc network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-759977d69f-rbtgb b2b06cb274a48965bc233adcac715996e29573ab1385a836821fca8b9a6647bc network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:a6:41:0c:98:4e\",\"name\":\"b2b06cb274a4896\"},{\"mac\":\"0a:58:0a:80:00:41\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2941220b-07ed-4a3d-a316-46322ea3cefb\"}],\"ips\":[{\"address\":\"10.128.0.65/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:06:24.102381 3885 cni.go:403] [release-service/release-service-controller-manager-854f775895-zw2g8 0714cec64b860fd8df7f3d7fbf7aff4e112a8923251a2d03fd0a0533f6571fca network default NAD default] ADD starting CNI request [release-service/release-service-controller-manager-854f775895-zw2g8 0714cec64b860fd8df7f3d7fbf7aff4e112a8923251a2d03fd0a0533f6571fca network default NAD default] I0329 03:06:24.114211 3885 helper_linux.go:450] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-854f775895-zw2g8, hostIfaceName: 0714cec64b860fd, network: default, NAD default, SandboxID: "0714cec64b860fd8df7f3d7fbf7aff4e112a8923251a2d03fd0a0533f6571fca", PCI device ID: , UID: "4877d74a-cb3d-4486-982b-d319c3a35036", MAC: 0a:58:0a:80:00:43, IPs: [10.128.0.67/23] I0329 03:06:24.176986 3885 cni.go:424] [release-service/release-service-controller-manager-854f775895-zw2g8 0714cec64b860fd8df7f3d7fbf7aff4e112a8923251a2d03fd0a0533f6571fca network default NAD default] ADD finished CNI request [release-service/release-service-controller-manager-854f775895-zw2g8 0714cec64b860fd8df7f3d7fbf7aff4e112a8923251a2d03fd0a0533f6571fca network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:70:9a:4e:b3:d1\",\"name\":\"0714cec64b860fd\"},{\"mac\":\"0a:58:0a:80:00:43\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b58a665c-6c83-4f25-84aa-604cc18e39be\"}],\"ips\":[{\"address\":\"10.128.0.67/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0329 03:06:25.334193 3885 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex I0329 03:06:25.685528 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0329 03:06:27.680332 3885 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 W0329 03:06:30.376504 3885 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0329 03:06:30.376523 3885 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0329 03:06:30.408430 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0329 03:06:30.408505 3885 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0329 03:06:30.425358 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0329 03:06:30.425821 3885 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0329 03:06:31.301129 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0329 03:06:31.331448 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0329 03:06:32.755769 3885 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0329 03:06:32.755791 3885 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0329 03:06:32.779007 3885 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0329 03:06:32.779034 3885 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0329 03:06:33.668695 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=11 I0329 03:06:33.674223 3885 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 W0329 03:06:41.381960 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0329 03:06:41.402704 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0329 03:06:41.410195 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0329 03:06:41.431697 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0329 03:06:41.431814 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0329 03:06:43.990958 3885 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-rbtgb b2b06cb274a48965bc233adcac715996e29573ab1385a836821fca8b9a6647bc network default NAD default] DEL starting CNI request [integration-service/integration-service-controller-manager-759977d69f-rbtgb b2b06cb274a48965bc233adcac715996e29573ab1385a836821fca8b9a6647bc network default NAD default] I0329 03:06:44.071591 3885 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-rbtgb b2b06cb274a48965bc233adcac715996e29573ab1385a836821fca8b9a6647bc network default NAD default] DEL finished CNI request [integration-service/integration-service-controller-manager-759977d69f-rbtgb b2b06cb274a48965bc233adcac715996e29573ab1385a836821fca8b9a6647bc network default NAD default], result "{}", err I0329 03:06:44.149641 3885 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-rbtgb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:44.149683 3885 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-759977d69f-rbtgb I0329 03:06:44.151172 3885 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-759977d69f-rbtgb, ips: 10.128.0.65 I0329 03:06:44.151201 3885 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-rbtgb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:06:44.470920 3885 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-rbtgb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:44.470947 3885 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-rbtgb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:06:44.480192 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-rbtgb I0329 03:06:44.480221 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-rbtgb I0329 03:06:52.558244 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=171 W0329 03:07:08.446006 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0329 03:07:08.446028 3885 util.go:959] Failed to build node endpoints for node ip-10-0-2-173.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy I0329 03:07:39.209577 3885 cni.go:403] [konflux-ui/proxy-5bbc66485b-k45f5 3b93e0284ebcadd47115a0cf7a82e6d6f6debd01e22896fe83192e0455511edb network default NAD default] DEL starting CNI request [konflux-ui/proxy-5bbc66485b-k45f5 3b93e0284ebcadd47115a0cf7a82e6d6f6debd01e22896fe83192e0455511edb network default NAD default] I0329 03:07:39.299350 3885 cni.go:424] [konflux-ui/proxy-5bbc66485b-k45f5 3b93e0284ebcadd47115a0cf7a82e6d6f6debd01e22896fe83192e0455511edb network default NAD default] DEL finished CNI request [konflux-ui/proxy-5bbc66485b-k45f5 3b93e0284ebcadd47115a0cf7a82e6d6f6debd01e22896fe83192e0455511edb network default NAD default], result "{}", err I0329 03:07:39.367348 3885 obj_retry.go:464] Detected object konflux-ui/proxy-5bbc66485b-k45f5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:07:39.367381 3885 pods.go:176] Deleting pod: konflux-ui/proxy-5bbc66485b-k45f5 I0329 03:07:39.368866 3885 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-5bbc66485b-k45f5, ips: 10.128.0.63 I0329 03:07:39.368896 3885 obj_retry.go:464] Detected object konflux-ui/proxy-5bbc66485b-k45f5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:07:39.826583 3885 obj_retry.go:458] Detected object konflux-ui/proxy-5bbc66485b-k45f5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:07:39.826609 3885 obj_retry.go:458] Detected object konflux-ui/proxy-5bbc66485b-k45f5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:07:39.835132 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-5bbc66485b-k45f5 I0329 03:07:39.835162 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-5bbc66485b-k45f5 I0329 03:07:54.331181 3885 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:07:54.331803 3885 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:07:54.352567 3885 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:07:54.352589 3885 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:07:54.357875 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0329 03:07:54.357896 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0329 03:08:19.198921 3885 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:08:19.199517 3885 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:08:19.223118 3885 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-56b684c87-v679l] creating logical port image-controller_image-controller-controller-manager-56b684c87-v679l for pod on switch ip-10-0-2-173.ec2.internal I0329 03:08:19.223206 3885 kube.go:256] Updating pod image-controller/image-controller-controller-manager-56b684c87-v679l I0329 03:08:19.241144 3885 pod.go:62] [image-controller/image-controller-controller-manager-56b684c87-v679l] pod update took 17.964701ms I0329 03:08:19.241169 3885 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-56b684c87-v679l] addLogicalPort annotation time took 17.992636ms I0329 03:08:19.242642 3885 pods.go:271] [image-controller/image-controller-controller-manager-56b684c87-v679l] addLogicalPort took 19.538535ms, libovsdb time 827.244µs I0329 03:08:19.588844 3885 cni.go:403] [image-controller/image-controller-controller-manager-56b684c87-v679l 2d2baba27ac37abc968d5b570019d226d244809732a3f2078573b050f452aacd network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-56b684c87-v679l 2d2baba27ac37abc968d5b570019d226d244809732a3f2078573b050f452aacd network default NAD default] I0329 03:08:19.597374 3885 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-56b684c87-v679l, hostIfaceName: 2d2baba27ac37ab, network: default, NAD default, SandboxID: "2d2baba27ac37abc968d5b570019d226d244809732a3f2078573b050f452aacd", PCI device ID: , UID: "304177da-2e45-4176-bc95-faa07c7a2b3b", MAC: 0a:58:0a:80:00:44, IPs: [10.128.0.68/23] I0329 03:08:19.645762 3885 cni.go:424] [image-controller/image-controller-controller-manager-56b684c87-v679l 2d2baba27ac37abc968d5b570019d226d244809732a3f2078573b050f452aacd network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-56b684c87-v679l 2d2baba27ac37abc968d5b570019d226d244809732a3f2078573b050f452aacd network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:36:a6:c6:71:0c\",\"name\":\"2d2baba27ac37ab\"},{\"mac\":\"0a:58:0a:80:00:44\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c526b21b-387a-4b78-9db7-94652b038ee2\"}],\"ips\":[{\"address\":\"10.128.0.68/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0329 03:08:19.694813 3885 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:08:19.694837 3885 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:08:19.702783 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 I0329 03:08:19.702811 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-fnfb6 I0329 03:08:21.557641 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0329 03:08:32.272707 3885 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz 9d6ccc148a886f469d1ea8066d7161710793271dbe6ce860b6c3d6020fb0af30 network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz 9d6ccc148a886f469d1ea8066d7161710793271dbe6ce860b6c3d6020fb0af30 network default NAD default] I0329 03:08:32.357872 3885 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz 9d6ccc148a886f469d1ea8066d7161710793271dbe6ce860b6c3d6020fb0af30 network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz 9d6ccc148a886f469d1ea8066d7161710793271dbe6ce860b6c3d6020fb0af30 network default NAD default], result "{}", err I0329 03:08:32.410795 3885 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0329 03:08:32.410831 3885 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz I0329 03:08:32.412137 3885 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz, ips: 10.128.0.64 I0329 03:08:32.412164 3885 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0329 03:08:33.157593 3885 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:08:33.157619 3885 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0329 03:08:33.166046 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz I0329 03:08:33.166095 3885 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-t5tlz I0329 03:08:42.563108 3885 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=678 I0329 03:08:46.647213 3885 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10