+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-1-27.ec2.internal ++ [[ -n ip-10-0-1-27.ec2.internal ]] ++ [[ -f /env/ip-10-0-1-27.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260311-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260311-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260311-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' I0330 14:59:43.873990703 - disable conntrack on geneve port + echo 'I0330 14:59:43.873990703 - disable conntrack on geneve port' + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0330 14:59:43.891495802 - starting ovnkube-node + echo 'I0330 14:59:43.891495802 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-1-27.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-1-27.ec2.internal --init-node ip-10-0-1-27.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-1-27.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0330 14:59:43.932029 3819 config.go:2459] Parsed config file /run/ovnkube-config/ovnkube.conf I0330 14:59:43.932079 3819 config.go:2460] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[]} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CAData:[] APIServer:https://api.kx-4a6a5c4963.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/}} I0330 14:59:43.933580 3819 kube.go:419] Waiting for certificate I0330 14:59:43.933629 3819 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0330 14:59:43.934031 3819 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0330 14:59:43.934718 3819 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0330 14:59:43.966995 3819 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0330 14:59:43.967016 3819 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0330 14:59:43.969847 3819 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0330 14:59:43.969936 3819 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-d5m5j" I0330 14:59:43.976574 3819 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-d5m5j" I0330 14:59:43.976640 3819 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0330 14:59:44.934400 3819 kube.go:426] Certificate found I0330 14:59:44.934988 3819 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0330 14:59:44.935912 3819 metrics.go:541] Starting metrics server at address "127.0.0.1:29103" I0330 14:59:44.980935 3819 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-03-31 14:54:43 +0000 UTC" deadline="2026-03-31 10:16:49.160819793 +0000 UTC" I0330 14:59:44.980971 3819 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h17m4.179859628s" I0330 14:59:45.003039 3819 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0330 14:59:45.015491 3819 metrics.go:541] Starting metrics server at address "127.0.0.1:29105" I0330 14:59:45.017055 3819 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0330 14:59:45.017217 3819 factory.go:531] Starting watch factory I0330 14:59:45.017349 3819 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.017400 3819 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.017447 3819 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.017478 3819 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.017497 3819 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.017919 3819 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.017981 3819 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.018164 3819 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.018283 3819 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.017377 3819 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.018775 3819 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.018796 3819 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.024494 3819 controller_manager.go:368] Starting the ovnkube controller I0330 14:59:45.024514 3819 controller_manager.go:373] Waiting up to 5m0s for NBDB zone to match: ip-10-0-1-27.ec2.internal I0330 14:59:45.024630 3819 controller_manager.go:393] NBDB zone sync took: 74.178µs I0330 14:59:45.024645 3819 factory.go:531] Starting watch factory I0330 14:59:45.038439 3819 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.038720 3819 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.039800 3819 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.040131 3819 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.041189 3819 ovn_northd.go:121] Registering OVN northd metrics I0330 14:59:45.041809 3819 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.041841 3819 ovn_db.go:368] Found OVN DB Pod running on this node. Registering OVN DB Metrics I0330 14:59:45.047307 3819 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0330 14:59:45.047338 3819 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0330 14:59:45.050953 3819 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0330 14:59:45.063021 3819 ovn_db.go:415] Found db is standalone, don't register db_cluster metrics I0330 14:59:45.122849 3819 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0330 14:59:45.122885 3819 factory.go:1890] *v1.Pod informer cache synced successfully I0330 14:59:45.122889 3819 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0330 14:59:45.122893 3819 factory.go:1890] *v1.Namespace informer cache synced successfully I0330 14:59:45.122896 3819 factory.go:1890] *v1.Node informer cache synced successfully I0330 14:59:45.122899 3819 factory.go:1890] *v1.Service informer cache synced successfully I0330 14:59:45.123000 3819 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0330 14:59:45.123019 3819 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0330 14:59:45.123066 3819 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0330 14:59:45.123100 3819 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0330 14:59:45.125850 3819 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0330 14:59:45.125846 3819 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0330 14:59:45.128886 3819 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0330 14:59:45.128895 3819 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0330 14:59:45.128956 3819 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.128969 3819 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.130607 3819 factory.go:1890] *v1.Service informer cache synced successfully I0330 14:59:45.130619 3819 factory.go:1890] *v1.EndpointSlice informer cache synced successfully I0330 14:59:45.130622 3819 factory.go:1890] *v1.Pod informer cache synced successfully I0330 14:59:45.130626 3819 factory.go:1890] *v1.NetworkPolicy informer cache synced successfully I0330 14:59:45.130629 3819 factory.go:1890] *v1.Namespace informer cache synced successfully I0330 14:59:45.130632 3819 factory.go:1890] *v1.Node informer cache synced successfully I0330 14:59:45.130895 3819 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.134833 3819 factory.go:1890] *v1.EgressIP informer cache synced successfully I0330 14:59:45.134895 3819 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.134939 3819 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.135668 3819 factory.go:1890] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0330 14:59:45.135680 3819 factory.go:1890] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0330 14:59:45.136953 3819 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.140714 3819 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0330 14:59:45.140778 3819 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.140790 3819 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.140808 3819 factory.go:1890] *v1.EgressIP informer cache synced successfully I0330 14:59:45.142801 3819 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.146595 3819 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0330 14:59:45.146610 3819 factory.go:1890] *v1.EgressFirewall informer cache synced successfully I0330 14:59:45.146693 3819 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.146720 3819 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.148853 3819 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.152565 3819 factory.go:1890] *v1.EgressService informer cache synced successfully I0330 14:59:45.152577 3819 factory.go:1890] *v1.EgressQoS informer cache synced successfully I0330 14:59:45.152632 3819 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.152644 3819 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.154497 3819 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.158463 3819 factory.go:1890] *v1.EgressService informer cache synced successfully I0330 14:59:45.158558 3819 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0330 14:59:45.158760 3819 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0330 14:59:45.158778 3819 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0330 14:59:45.160999 3819 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0330 14:59:45.163934 3819 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0330 14:59:45.163942 3819 factory.go:1890] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0330 14:59:45.164006 3819 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.164019 3819 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.164064 3819 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.164085 3819 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.166432 3819 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.166746 3819 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0330 14:59:45.169266 3819 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0330 14:59:45.169278 3819 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0330 14:59:45.169287 3819 factory.go:636] Watch Factory start up complete, took: 152.07416ms I0330 14:59:45.169364 3819 default_node_network_controller.go:170] Enable node proxy healthz server on 0.0.0.0:10256 I0330 14:59:45.169446 3819 factory.go:1890] *v1.NetworkAttachmentDefinition informer cache synced successfully I0330 14:59:45.174835 3819 factory.go:1890] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0330 14:59:45.174845 3819 factory.go:1890] *v1.UserDefinedNetwork informer cache synced successfully I0330 14:59:45.174851 3819 factory.go:636] Watch Factory start up complete, took: 150.208009ms I0330 14:59:45.174858 3819 controller_manager.go:404] Waiting up to 5m0s for a node to have "ip-10-0-1-27.ec2.internal" zone I0330 14:59:45.209097 3819 default_node_network_controller.go:709] Initializing the default node network controller I0330 14:59:45.258728 3819 udn_isolation.go:101] Starting UDN host isolation manager I0330 14:59:45.264860 3819 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0330 14:59:45.310458 3819 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0330 14:59:45.310575 3819 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0330 14:59:45.310589 3819 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0330 14:59:45.345601 3819 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0330 14:59:45.345676 3819 default_node_network_controller.go:808] Node ip-10-0-1-27.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0330 14:59:45.351347 3819 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.1.27"] k8s.ovn.org/zone-name:ip-10-0-1-27.ec2.internal] on node ip-10-0-1-27.ec2.internal I0330 14:59:45.372659 3819 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0330 14:59:45.378236 3819 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0330 14:59:45.378413 3819 helper_linux.go:118] Found default gateway interface br-ex 10.0.1.1 I0330 14:59:45.378575 3819 gateway_init.go:263] Preparing Gateway I0330 14:59:45.378587 3819 gateway_shared_intf.go:1658] Creating new gateway I0330 14:59:45.425002 3819 controller_manager.go:427] Waiting for node in zone sync took: 250.131624ms I0330 14:59:45.429384 3819 controller_manager.go:608] Switching to transit router for layer2 networks I0330 14:59:45.429404 3819 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-1-27.ec2.internal I0330 14:59:45.443333 3819 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0330 14:59:45.448882 3819 controller_manager.go:288] SCTP support detected in OVN I0330 14:59:45.449491 3819 services_controller.go:69] Creating services controller for network=default I0330 14:59:45.449588 3819 udn_enabled_svc.go:80] Waiting for service informer to sync I0330 14:59:45.449672 3819 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0330 14:59:45.449701 3819 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0330 14:59:45.449713 3819 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0330 14:59:45.449725 3819 nad_controller.go:241] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0330 14:59:45.449895 3819 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0330 14:59:45.449944 3819 network_controller.go:246] [zone-nad-controller network controller]: syncing all networks I0330 14:59:45.449979 3819 nad_controller.go:280] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 54.002µs I0330 14:59:45.449998 3819 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 45.522µs I0330 14:59:45.450005 3819 network_controller.go:257] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 61.145µs I0330 14:59:45.450010 3819 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0330 14:59:45.450015 3819 nad_controller.go:173] [zone-nad-controller NAD controller]: started I0330 14:59:45.450024 3819 default_network_controller.go:321] Starting the default network controller I0330 14:59:45.450051 3819 udn_enabled_svc.go:95] Performing full resync I0330 14:59:45.450106 3819 network_controller.go:275] [zone-nad-controller network controller]: finished syncing network default, took 12.169µs I0330 14:59:45.450376 3819 udn_enabled_svc.go:99] Waiting for handler to sync I0330 14:59:45.450387 3819 udn_enabled_svc.go:104] Starting worker I0330 14:59:45.455868 3819 iptables.go:108] Creating table: filter chain: FORWARD I0330 14:59:45.458684 3819 default_network_controller.go:388] Cleaning External Gateway ECMP routes I0330 14:59:45.458745 3819 repair.go:33] Syncing exgw routes took 47.752µs I0330 14:59:45.458765 3819 default_network_controller.go:407] Starting all the Watchers... I0330 14:59:45.458904 3819 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0330 14:59:45.458911 3819 namespace.go:98] [openshift-console-user-settings] adding namespace I0330 14:59:45.458926 3819 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0330 14:59:45.458933 3819 namespace.go:98] [openshift-infra] adding namespace I0330 14:59:45.458945 3819 namespace.go:98] [openshift-must-gather-operator] adding namespace I0330 14:59:45.458950 3819 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0330 14:59:45.458951 3819 namespace.go:98] [openshift-multus] adding namespace I0330 14:59:45.458944 3819 namespace.go:98] [kube-public] adding namespace I0330 14:59:45.458977 3819 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0330 14:59:45.458985 3819 namespace.go:98] [openshift-network-console] adding namespace I0330 14:59:45.458993 3819 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0330 14:59:45.458981 3819 namespace.go:98] [openshift-machine-api] adding namespace I0330 14:59:45.459035 3819 namespace.go:98] [openshift-config-operator] adding namespace I0330 14:59:45.459042 3819 namespace.go:98] [openshift-console] adding namespace I0330 14:59:45.459070 3819 namespace.go:98] [kube-node-lease] adding namespace I0330 14:59:45.459480 3819 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 563.763µs I0330 14:59:45.459494 3819 namespace.go:98] [openshift-controller-manager] adding namespace I0330 14:59:45.460072 3819 namespace.go:102] [openshift-console-user-settings] adding namespace took 1.150773ms I0330 14:59:45.460084 3819 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0330 14:59:45.460554 3819 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 442.325µs I0330 14:59:45.460569 3819 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0330 14:59:45.461024 3819 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 2.088737ms I0330 14:59:45.461041 3819 namespace.go:98] [openshift-kube-scheduler] adding namespace I0330 14:59:45.461417 3819 namespace.go:102] [openshift-infra] adding namespace took 2.478921ms I0330 14:59:45.461431 3819 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0330 14:59:45.461977 3819 namespace.go:102] [openshift-must-gather-operator] adding namespace took 3.021401ms I0330 14:59:45.461992 3819 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0330 14:59:45.462447 3819 iptables.go:108] Creating table: filter chain: OUTPUT I0330 14:59:45.462457 3819 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 3.499934ms I0330 14:59:45.462468 3819 namespace.go:98] [default] adding namespace I0330 14:59:45.463006 3819 namespace.go:102] [kube-public] adding namespace took 4.035812ms I0330 14:59:45.463015 3819 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace W0330 14:59:45.463038 3819 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-mjzq2: pod openshift-multus/network-metrics-daemon-mjzq2: no pod IPs found W0330 14:59:45.463052 3819 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-multus/network-metrics-daemon-sj6h6: pod openshift-multus/network-metrics-daemon-sj6h6: no pod IPs found I0330 14:59:45.463565 3819 namespace.go:102] [openshift-multus] adding namespace took 4.593486ms I0330 14:59:45.463582 3819 namespace.go:98] [openshift-host-network] adding namespace I0330 14:59:45.464045 3819 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 5.054707ms I0330 14:59:45.464059 3819 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0330 14:59:45.464564 3819 namespace.go:102] [openshift-network-console] adding namespace took 5.565678ms I0330 14:59:45.464575 3819 namespace.go:98] [openshift-dns] adding namespace I0330 14:59:45.465647 3819 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.642024ms I0330 14:59:45.465661 3819 namespace.go:98] [open-cluster-management-2pc3srqfa75va1b7r65hvfp4e15g46er] adding namespace I0330 14:59:45.466063 3819 namespace.go:102] [openshift-machine-api] adding namespace took 7.053268ms I0330 14:59:45.466079 3819 namespace.go:98] [openshift-config-managed] adding namespace I0330 14:59:45.466572 3819 namespace.go:102] [openshift-config-operator] adding namespace took 7.528357ms I0330 14:59:45.466588 3819 namespace.go:98] [openshift-backplane-csm] adding namespace I0330 14:59:45.467098 3819 namespace.go:102] [openshift-console] adding namespace took 8.043132ms I0330 14:59:45.467112 3819 namespace.go:98] [openshift-marketplace] adding namespace I0330 14:59:45.467549 3819 namespace.go:102] [kube-node-lease] adding namespace took 8.464816ms I0330 14:59:45.467558 3819 namespace.go:98] [openshift-backplane-cee] adding namespace I0330 14:59:45.468130 3819 namespace.go:102] [openshift-controller-manager] adding namespace took 8.631335ms I0330 14:59:45.468139 3819 namespace.go:98] [openshift-etcd] adding namespace I0330 14:59:45.468616 3819 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 8.040901ms I0330 14:59:45.468629 3819 namespace.go:98] [openshift-package-operator] adding namespace I0330 14:59:45.468808 3819 gateway_shared_intf.go:1787] Gateway Creation Complete I0330 14:59:45.468822 3819 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-27.ec2.internal","mac-address":"0a:ff:f8:38:b4:23","ip-addresses":["10.0.1.27/24"],"ip-address":"10.0.1.27/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:b79800b8-57d8-4cdc-a2b4-bb13c2b69488 k8s.ovn.org/node-encap-ips:["10.0.1.27"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.27/24"} k8s.ovn.org/zone-name:ip-10-0-1-27.ec2.internal] on node ip-10-0-1-27.ec2.internal I0330 14:59:45.468986 3819 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.939506ms I0330 14:59:45.468998 3819 namespace.go:98] [openshift-backplane-srep] adding namespace I0330 14:59:45.469470 3819 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 8.032393ms I0330 14:59:45.469483 3819 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0330 14:59:45.469924 3819 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.927355ms I0330 14:59:45.469937 3819 namespace.go:98] [openshift-operators] adding namespace I0330 14:59:45.470317 3819 namespace.go:102] [default] adding namespace took 7.843884ms I0330 14:59:45.470330 3819 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0330 14:59:45.470694 3819 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.674127ms I0330 14:59:45.470706 3819 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0330 14:59:45.471283 3819 namespace.go:102] [openshift-host-network] adding namespace took 7.696002ms I0330 14:59:45.471297 3819 namespace.go:98] [openshift-backplane-cse] adding namespace I0330 14:59:45.471650 3819 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.587189ms I0330 14:59:45.471663 3819 namespace.go:98] [openshift-dns-operator] adding namespace I0330 14:59:45.472177 3819 namespace.go:102] [openshift-dns] adding namespace took 7.596577ms I0330 14:59:45.472191 3819 namespace.go:98] [openshift-operators-redhat] adding namespace I0330 14:59:45.472584 3819 namespace.go:102] [open-cluster-management-2pc3srqfa75va1b7r65hvfp4e15g46er] adding namespace took 6.918067ms I0330 14:59:45.472596 3819 namespace.go:98] [openshift-ingress-canary] adding namespace I0330 14:59:45.472995 3819 namespace.go:102] [openshift-config-managed] adding namespace took 6.911687ms I0330 14:59:45.473007 3819 namespace.go:98] [openshift-image-registry] adding namespace I0330 14:59:45.473321 3819 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.726553ms I0330 14:59:45.473338 3819 namespace.go:98] [openshift-monitoring] adding namespace I0330 14:59:45.473822 3819 namespace.go:102] [openshift-marketplace] adding namespace took 6.704931ms I0330 14:59:45.473834 3819 namespace.go:98] [openshift-network-node-identity] adding namespace I0330 14:59:45.474182 3819 namespace.go:102] [openshift-backplane-cee] adding namespace took 6.619992ms I0330 14:59:45.474193 3819 namespace.go:98] [openshift-ingress-operator] adding namespace I0330 14:59:45.474554 3819 namespace.go:102] [openshift-etcd] adding namespace took 6.41046ms I0330 14:59:45.474565 3819 namespace.go:98] [dedicated-admin] adding namespace I0330 14:59:45.475035 3819 namespace.go:102] [openshift-package-operator] adding namespace took 6.40148ms I0330 14:59:45.475048 3819 namespace.go:98] [openshift-authentication-operator] adding namespace I0330 14:59:45.475437 3819 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.434522ms I0330 14:59:45.475447 3819 namespace.go:98] [openshift-machine-config-operator] adding namespace I0330 14:59:45.475860 3819 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.372596ms I0330 14:59:45.475873 3819 namespace.go:98] [openshift] adding namespace I0330 14:59:45.476192 3819 namespace.go:102] [openshift-operators] adding namespace took 6.249538ms I0330 14:59:45.476208 3819 namespace.go:98] [openshift-kube-apiserver] adding namespace I0330 14:59:45.476648 3819 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.313407ms I0330 14:59:45.476660 3819 namespace.go:98] [openshift-service-ca-operator] adding namespace I0330 14:59:45.477037 3819 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 6.326526ms I0330 14:59:45.477049 3819 namespace.go:98] [openshift-backplane-tam] adding namespace I0330 14:59:45.477417 3819 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.116162ms I0330 14:59:45.477428 3819 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0330 14:59:45.477827 3819 namespace.go:102] [openshift-dns-operator] adding namespace took 6.160063ms I0330 14:59:45.477839 3819 namespace.go:98] [openshift-logging] adding namespace I0330 14:59:45.478161 3819 namespace.go:102] [openshift-operators-redhat] adding namespace took 5.965708ms I0330 14:59:45.478169 3819 namespace.go:98] [openshift-customer-monitoring] adding namespace I0330 14:59:45.478687 3819 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.084853ms I0330 14:59:45.478700 3819 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0330 14:59:45.479176 3819 namespace.go:102] [openshift-image-registry] adding namespace took 6.163348ms I0330 14:59:45.479190 3819 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0330 14:59:45.479555 3819 namespace.go:102] [openshift-monitoring] adding namespace took 6.213155ms I0330 14:59:45.479567 3819 namespace.go:98] [openshift-backplane-mobb] adding namespace I0330 14:59:45.479971 3819 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.130936ms I0330 14:59:45.479984 3819 namespace.go:98] [openshift-network-operator] adding namespace I0330 14:59:45.480413 3819 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.215231ms I0330 14:59:45.480423 3819 namespace.go:98] [openshift-apiserver] adding namespace I0330 14:59:45.480811 3819 namespace.go:102] [dedicated-admin] adding namespace took 6.24156ms I0330 14:59:45.480824 3819 namespace.go:98] [openshift-insights] adding namespace I0330 14:59:45.481258 3819 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.205029ms I0330 14:59:45.481643 3819 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.191936ms I0330 14:59:45.481654 3819 namespace.go:98] [openshift-console-operator] adding namespace I0330 14:59:45.482116 3819 namespace.go:102] [openshift] adding namespace took 6.237528ms I0330 14:59:45.482129 3819 namespace.go:98] [openshift-config] adding namespace I0330 14:59:45.482508 3819 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.296117ms I0330 14:59:45.482520 3819 namespace.go:98] [openshift-authentication] adding namespace I0330 14:59:45.482981 3819 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.317091ms I0330 14:59:45.482993 3819 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0330 14:59:45.483365 3819 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.310566ms I0330 14:59:45.483378 3819 namespace.go:98] [openshift-backplane] adding namespace I0330 14:59:45.483818 3819 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.382434ms I0330 14:59:45.483841 3819 namespace.go:98] [openshift-node] adding namespace I0330 14:59:45.484447 3819 namespace.go:102] [openshift-logging] adding namespace took 6.601987ms I0330 14:59:45.484465 3819 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0330 14:59:45.484924 3819 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.749131ms I0330 14:59:45.485363 3819 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.657227ms I0330 14:59:45.485794 3819 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.599049ms I0330 14:59:45.486218 3819 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.646441ms I0330 14:59:45.486229 3819 namespace.go:98] [openshift-ingress] adding namespace I0330 14:59:45.486635 3819 namespace.go:102] [openshift-network-operator] adding namespace took 6.644605ms I0330 14:59:45.486651 3819 namespace.go:98] [kube-system] adding namespace I0330 14:59:45.487023 3819 namespace.go:102] [openshift-apiserver] adding namespace took 6.595004ms I0330 14:59:45.487036 3819 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0330 14:59:45.487193 3819 default_node_network_controller.go:518] Node connection status = connected I0330 14:59:45.487618 3819 namespace.go:102] [openshift-insights] adding namespace took 6.788431ms I0330 14:59:45.488128 3819 namespace.go:102] [openshift-console-operator] adding namespace took 6.467551ms I0330 14:59:45.488141 3819 namespace.go:98] [openshift-route-controller-manager] adding namespace I0330 14:59:45.488501 3819 namespace.go:102] [openshift-config] adding namespace took 6.36621ms I0330 14:59:45.488515 3819 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0330 14:59:45.488924 3819 namespace.go:102] [openshift-authentication] adding namespace took 6.399566ms I0330 14:59:45.488939 3819 namespace.go:98] [openshift-apiserver-operator] adding namespace I0330 14:59:45.489366 3819 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.367028ms I0330 14:59:45.489789 3819 namespace.go:102] [openshift-backplane] adding namespace took 6.405141ms I0330 14:59:45.489803 3819 namespace.go:98] [openshift-network-diagnostics] adding namespace I0330 14:59:45.490283 3819 namespace.go:102] [openshift-node] adding namespace took 6.434826ms I0330 14:59:45.490707 3819 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 6.237386ms I0330 14:59:45.490719 3819 namespace.go:98] [openshift-cluster-version] adding namespace W0330 14:59:45.491166 3819 base_network_controller_namespace.go:409] Failed to get IPs for pod kube-system/global-pull-secret-syncer-hft9h: pod kube-system/global-pull-secret-syncer-hft9h: no pod IPs found I0330 14:59:45.491215 3819 namespace.go:102] [openshift-ingress] adding namespace took 4.980097ms I0330 14:59:45.491229 3819 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0330 14:59:45.491942 3819 namespace.go:102] [kube-system] adding namespace took 5.28661ms I0330 14:59:45.491954 3819 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0330 14:59:45.492395 3819 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 5.354473ms I0330 14:59:45.492907 3819 namespace.go:102] [openshift-route-controller-manager] adding namespace took 4.761213ms I0330 14:59:45.493497 3819 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 4.976974ms W0330 14:59:45.493919 3819 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-c2f7q: pod openshift-network-diagnostics/network-check-target-c2f7q: no pod IPs found W0330 14:59:45.493933 3819 base_network_controller_namespace.go:409] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-vtznl: pod openshift-network-diagnostics/network-check-target-vtznl: no pod IPs found I0330 14:59:45.493950 3819 namespace.go:102] [openshift-apiserver-operator] adding namespace took 5.007032ms I0330 14:59:45.494428 3819 namespace.go:102] [openshift-network-diagnostics] adding namespace took 4.618821ms I0330 14:59:45.494881 3819 namespace.go:102] [openshift-cluster-version] adding namespace took 4.15781ms I0330 14:59:45.495287 3819 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 4.054797ms I0330 14:59:45.495646 3819 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 3.686216ms I0330 14:59:45.496017 3819 zone_ic_handler.go:186] Time taken to create transit switch: 268.998µs I0330 14:59:45.496103 3819 master.go:532] Adding or Updating local node "ip-10-0-1-27.ec2.internal" for network "default" I0330 14:59:45.496130 3819 obj_retry.go:544] Failed to create *v1.Node ip-10-0-0-65.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-65.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-65.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-65.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-65.ec2.internal I0330 14:59:45.496767 3819 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-2-167.ec2.internal for the network default I0330 14:59:45.497920 3819 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-2-167.ec2.internal" on network "default" took: 1.160635ms W0330 14:59:45.506677 3819 gateway.go:197] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-sj6h6: pod openshift-multus/network-metrics-daemon-sj6h6: no pod IPs found W0330 14:59:45.506704 3819 gateway.go:197] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-c2f7q: pod openshift-network-diagnostics/network-check-target-c2f7q: no pod IPs found W0330 14:59:45.506713 3819 gateway.go:197] Unable to fetch podIPs for pod kube-system/global-pull-secret-syncer-hft9h: pod kube-system/global-pull-secret-syncer-hft9h: no pod IPs found I0330 14:59:45.507964 3819 zone_ic_handler.go:194] Creating interconnect resources for local zone node ip-10-0-1-27.ec2.internal for the network default I0330 14:59:45.509016 3819 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0330 14:59:45.509059 3819 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0330 14:59:45.509082 3819 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0330 14:59:45.509102 3819 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-65.ec2.internal in network "default" I0330 14:59:45.509170 3819 node_tracker.go:235] Node ip-10-0-0-65.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-65.ec2.internal" I0330 14:59:45.509187 3819 node_tracker.go:169] Node ip-10-0-0-65.ec2.internal switch + router changed, syncing services in network "default" I0330 14:59:45.509199 3819 services_controller.go:574] Full service sync requested for network=default I0330 14:59:45.509210 3819 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-27.ec2.internal in network "default" I0330 14:59:45.509256 3819 node_tracker.go:169] Node ip-10-0-1-27.ec2.internal switch + router changed, syncing services in network "default" I0330 14:59:45.509266 3819 services_controller.go:574] Full service sync requested for network=default I0330 14:59:45.509275 3819 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-167.ec2.internal in network "default" I0330 14:59:45.509309 3819 node_tracker.go:169] Node ip-10-0-2-167.ec2.internal switch + router changed, syncing services in network "default" I0330 14:59:45.509317 3819 services_controller.go:574] Full service sync requested for network=default I0330 14:59:45.609862 3819 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0330 14:59:45.609890 3819 services_controller.go:208] Setting up event handlers for services for network=default I0330 14:59:45.610004 3819 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0330 14:59:45.610132 3819 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0330 14:59:45.610156 3819 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0330 14:59:45.710421 3819 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0330 14:59:45.710446 3819 repair.go:60] Starting repairing loop for services I0330 14:59:45.710570 3819 repair.go:131] Deleted 0 stale service LBs I0330 14:59:45.710596 3819 repair.go:137] Deleted 0 stale Chassis Template Vars I0330 14:59:45.710617 3819 repair.go:62] Finished repairing loop for services: 173.213µs I0330 14:59:45.710642 3819 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0330 14:59:45.710653 3819 services_controller.go:254] Starting workers for network=default I0330 14:59:45.711419 3819 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-c2f7q] creating logical port openshift-network-diagnostics_network-check-target-c2f7q for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:45.711477 3819 obj_retry.go:544] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-vtznl, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-vtznl: suppressed error logged: pod openshift-network-diagnostics/network-check-target-vtznl: no pod IPs found I0330 14:59:45.711592 3819 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-sj6h6] creating logical port openshift-multus_network-metrics-daemon-sj6h6 for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:45.711648 3819 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-sf9l4 of type *v1.Pod in terminal state (e.g. completed) during add event: will remove it I0330 14:59:45.711666 3819 obj_retry.go:544] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-mjzq2, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-mjzq2: suppressed error logged: pod openshift-multus/network-metrics-daemon-mjzq2: no pod IPs found I0330 14:59:45.711740 3819 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-sj6h6 I0330 14:59:45.711740 3819 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-c2f7q I0330 14:59:45.711594 3819 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-hft9h] creating logical port kube-system_global-pull-secret-syncer-hft9h for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:45.712512 3819 kube.go:256] Updating pod kube-system/global-pull-secret-syncer-hft9h W0330 14:59:45.716727 3819 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0330 14:59:45.727428 3819 pod.go:62] [kube-system/global-pull-secret-syncer-hft9h] pod update took 15.586738ms I0330 14:59:45.727458 3819 base_network_controller_pods.go:937] [default/kube-system/global-pull-secret-syncer-hft9h] addLogicalPort annotation time took 15.625667ms I0330 14:59:45.728424 3819 pods.go:271] [kube-system/global-pull-secret-syncer-hft9h] addLogicalPort took 16.836726ms, libovsdb time 694.357µs I0330 14:59:45.729187 3819 pod.go:62] [openshift-network-diagnostics/network-check-target-c2f7q] pod update took 17.581929ms I0330 14:59:45.729247 3819 base_network_controller_pods.go:937] [default/openshift-network-diagnostics/network-check-target-c2f7q] addLogicalPort annotation time took 17.645373ms I0330 14:59:45.730258 3819 pods.go:271] [openshift-network-diagnostics/network-check-target-c2f7q] addLogicalPort took 18.86693ms, libovsdb time 742.719µs I0330 14:59:45.731648 3819 pod.go:62] [openshift-multus/network-metrics-daemon-sj6h6] pod update took 19.94582ms I0330 14:59:45.731667 3819 base_network_controller_pods.go:937] [default/openshift-multus/network-metrics-daemon-sj6h6] addLogicalPort annotation time took 19.967089ms I0330 14:59:45.732523 3819 pods.go:271] [openshift-multus/network-metrics-daemon-sj6h6] addLogicalPort took 20.944976ms, libovsdb time 668.475µs I0330 14:59:45.733079 3819 admin_network_policy_controller.go:227] Starting controller default-network-controller I0330 14:59:45.733106 3819 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0330 14:59:45.733122 3819 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0330 14:59:45.733127 3819 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0330 14:59:45.733367 3819 repair.go:29] Repairing admin network policies took 233.731µs I0330 14:59:45.733565 3819 repair.go:92] Repairing baseline admin network policies took 184.524µs I0330 14:59:45.734119 3819 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0330 14:59:45.734149 3819 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0330 14:59:45.737704 3819 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0330 14:59:45.737734 3819 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0330 14:59:45.737747 3819 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0330 14:59:45.740393 3819 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0330 14:59:45.740425 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0330 14:59:45.741459 3819 pod_selector_address_set.go:212] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0330 14:59:45.741479 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0330 14:59:45.743618 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0330 14:59:45.743639 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0330 14:59:45.743650 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0330 14:59:45.745360 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0330 14:59:45.745382 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0330 14:59:45.745412 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0330 14:59:45.746231 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0330 14:59:45.746259 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0330 14:59:45.746272 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0330 14:59:45.746865 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0330 14:59:45.746886 3819 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0330 14:59:45.746897 3819 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0330 14:59:45.748030 3819 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0330 14:59:45.748055 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0330 14:59:45.748067 3819 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0330 14:59:45.749982 3819 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0330 14:59:45.750007 3819 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0330 14:59:45.750021 3819 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0330 14:59:45.752565 3819 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0330 14:59:45.752592 3819 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0330 14:59:45.752600 3819 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0330 14:59:45.752993 3819 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0330 14:59:45.753017 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0330 14:59:45.753027 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0330 14:59:45.754057 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0330 14:59:45.754077 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0330 14:59:45.754085 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0330 14:59:45.754736 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0330 14:59:45.754755 3819 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0330 14:59:45.754764 3819 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0330 14:59:45.755955 3819 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0330 14:59:45.755979 3819 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0330 14:59:45.755999 3819 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0330 14:59:45.756743 3819 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0330 14:59:45.756760 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0330 14:59:45.756771 3819 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0330 14:59:45.757476 3819 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0330 14:59:45.757491 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0330 14:59:45.757499 3819 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0330 14:59:45.757967 3819 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0330 14:59:45.757987 3819 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0330 14:59:45.757999 3819 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0330 14:59:45.758988 3819 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0330 14:59:45.759006 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0330 14:59:45.759012 3819 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0330 14:59:45.759311 3819 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0330 14:59:45.759331 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0330 14:59:45.759339 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0330 14:59:45.759916 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0330 14:59:45.759936 3819 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0330 14:59:45.759943 3819 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0330 14:59:45.760249 3819 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0330 14:59:45.760266 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0330 14:59:45.760275 3819 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0330 14:59:45.760749 3819 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0330 14:59:45.760766 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0330 14:59:45.760774 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0330 14:59:45.761035 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0330 14:59:45.761052 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0330 14:59:45.761059 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0330 14:59:45.762254 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0330 14:59:45.762271 3819 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0330 14:59:45.762278 3819 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0330 14:59:45.762786 3819 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0330 14:59:45.762799 3819 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0330 14:59:45.762809 3819 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0330 14:59:45.763959 3819 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0330 14:59:45.763978 3819 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0330 14:59:45.763989 3819 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0330 14:59:45.764989 3819 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0330 14:59:45.765007 3819 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0330 14:59:45.765016 3819 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0330 14:59:45.766584 3819 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0330 14:59:45.766607 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0330 14:59:45.766619 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0330 14:59:45.767630 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0330 14:59:45.767649 3819 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0330 14:59:45.767664 3819 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0330 14:59:45.768152 3819 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0330 14:59:45.768404 3819 obj_retry.go:464] Detected object kube-system/ecr-credential-refresh-initial-sf9l4 of type *factory.egressIPPod in terminal state (e.g. completed) during add event: will remove it I0330 14:59:45.773466 3819 egressfirewall.go:319] Starting EgressFirewall controller I0330 14:59:45.773496 3819 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0330 14:59:45.773517 3819 controller.go:133] Adding controller egress-firewall-controller event handlers I0330 14:59:45.773526 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0330 14:59:45.773555 3819 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0330 14:59:45.773555 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0330 14:59:45.773565 3819 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0330 14:59:45.773841 3819 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0330 14:59:45.773855 3819 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0330 14:59:45.773865 3819 egressqos.go:193] Setting up event handlers for EgressQoS I0330 14:59:45.773937 3819 egressfirewall.go:628] Syncing node "ip-10-0-0-65.ec2.internal" for egress firewall I0330 14:59:45.773950 3819 egressfirewall.go:628] Syncing node "ip-10-0-1-27.ec2.internal" for egress firewall I0330 14:59:45.773955 3819 egressfirewall.go:628] Syncing node "ip-10-0-2-167.ec2.internal" for egress firewall I0330 14:59:45.774088 3819 egressqos.go:245] Starting EgressQoS Controller I0330 14:59:45.774102 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0330 14:59:45.774110 3819 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0330 14:59:45.774117 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0330 14:59:45.774122 3819 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0330 14:59:45.774128 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0330 14:59:45.774133 3819 shared_informer.go:356] "Caches are synced" controller="egressqos" I0330 14:59:45.774137 3819 egressqos.go:259] Repairing EgressQoSes I0330 14:59:45.774141 3819 egressqos.go:399] Starting repairing loop for egressqos I0330 14:59:45.774221 3819 egressqos.go:401] Finished repairing loop for egressqos: 78.841µs I0330 14:59:45.774240 3819 egressservice_zone.go:132] Setting up event handlers for Egress Services I0330 14:59:45.774271 3819 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-1-27.ec2.internal I0330 14:59:45.774335 3819 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-1-27.ec2.internal : 66.495µs I0330 14:59:45.774409 3819 egressservice_zone.go:210] Starting Egress Services Controller I0330 14:59:45.774418 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0330 14:59:45.774423 3819 shared_informer.go:356] "Caches are synced" controller="egressservices" I0330 14:59:45.774430 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0330 14:59:45.774434 3819 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0330 14:59:45.774440 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0330 14:59:45.774446 3819 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0330 14:59:45.774451 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0330 14:59:45.774455 3819 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0330 14:59:45.774459 3819 egressservice_zone.go:228] Repairing Egress Services I0330 14:59:45.774737 3819 master_controller.go:88] Starting Admin Policy Based Route Controller I0330 14:59:45.774748 3819 external_controller.go:278] Starting Admin Policy Based Route Controller I0330 14:59:45.774810 3819 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-65.ec2.internal I0330 14:59:45.774821 3819 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-65.ec2.internal: 12.29µs I0330 14:59:45.774830 3819 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-27.ec2.internal I0330 14:59:45.774833 3819 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-27.ec2.internal: 4.371µs I0330 14:59:45.774838 3819 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-167.ec2.internal I0330 14:59:45.774847 3819 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-167.ec2.internal: 9.594µs I0330 14:59:45.774876 3819 default_network_controller.go:568] Completing all the Watchers took 316.103334ms I0330 14:59:45.774888 3819 default_network_controller.go:572] Starting unidling controllers I0330 14:59:45.774900 3819 unidle.go:46] Registering OVN SB ControllerEvent handler I0330 14:59:45.774906 3819 unidle.go:63] Populating Initial ContollerEvent events I0330 14:59:45.774915 3819 unidle.go:79] Setting up event handlers for services I0330 14:59:45.774973 3819 ovnkube.go:544] Waiting for OVN northbound database changes to be processed by ovn-controller I0330 14:59:45.987809 3819 default_node_network_controller.go:518] Node connection status = connected I0330 14:59:46.487773 3819 default_node_network_controller.go:518] Node connection status = connected I0330 14:59:46.846286 3819 ovnkube.go:549] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0330 14:59:46.989184 3819 default_node_network_controller.go:518] Node connection status = connected I0330 14:59:47.005468 3819 bridgeconfig.go:537] Gateway is ready I0330 14:59:47.005489 3819 gateway_shared_intf.go:1704] Creating Gateway Openflow Manager I0330 14:59:47.016827 3819 node_ip_handler_linux.go:247] Node primary address changed to 10.0.1.27. Updating OVN encap IP. I0330 14:59:47.022481 3819 node_ip_handler_linux.go:542] Will not update encap IP 10.0.1.27 - it is already configured I0330 14:59:47.022503 3819 node_ip_handler_linux.go:485] Node address changed to map[10.0.1.27/24:{} 172.20.0.1/32:{}]. Updating annotations. I0330 14:59:47.022810 3819 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.1.27/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-1-27.ec2.internal","mac-address":"0a:ff:f8:38:b4:23","ip-addresses":["10.0.1.27/24"],"ip-address":"10.0.1.27/24","next-hops":["10.0.1.1"],"next-hop":"10.0.1.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:b79800b8-57d8-4cdc-a2b4-bb13c2b69488 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.1.27/24"}] on node ip-10-0-1-27.ec2.internal I0330 14:59:47.037334 3819 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-27.ec2.internal I0330 14:59:47.037358 3819 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-27.ec2.internal: 31.626µs I0330 14:59:47.037418 3819 master.go:532] Adding or Updating local node "ip-10-0-1-27.ec2.internal" for network "default" I0330 14:59:47.037432 3819 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-27.ec2.internal in network "default" I0330 14:59:47.037482 3819 node_tracker.go:169] Node ip-10-0-1-27.ec2.internal switch + router changed, syncing services in network "default" I0330 14:59:47.037498 3819 services_controller.go:574] Full service sync requested for network=default I0330 14:59:47.037420 3819 egressfirewall.go:628] Syncing node "ip-10-0-1-27.ec2.internal" for egress firewall I0330 14:59:47.037958 3819 gateway_shared_intf.go:1994] Setting OVN Masquerade route with source: 10.0.1.27 W0330 14:59:47.038029 3819 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0330 14:59:47.038036 3819 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-1-27.ec2.internal I0330 14:59:47.042022 3819 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-1-27.ec2.internal. Updating no re-route policies I0330 14:59:47.042090 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-mjzq2 I0330 14:59:47.042116 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-4v7ww I0330 14:59:47.042122 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-mjzq2 I0330 14:59:47.042131 3819 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/konnectivity-agent-kw8v7 I0330 14:59:47.042133 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-sj6h6 I0330 14:59:47.042143 3819 obj_retry.go:364] Adding new object: *v1.Pod kube-system/konnectivity-agent-kw8v7 I0330 14:59:47.042145 3819 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/global-pull-secret-syncer-hft9h I0330 14:59:47.042150 3819 obj_retry.go:300] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-27.ec2.internal I0330 14:59:47.042160 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-dns/node-resolver-458lx I0330 14:59:47.042163 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-svxb4 I0330 14:59:47.042166 3819 obj_retry.go:384] Retry successful for *v1.Pod kube-system/konnectivity-agent-kw8v7 after 0 failed attempt(s) I0330 14:59:47.042173 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-dns/node-resolver-458lx I0330 14:59:47.042172 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-vpbs4 I0330 14:59:47.042184 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-vpbs4 I0330 14:59:47.042182 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-p9wnr I0330 14:59:47.042184 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-image-registry/node-ca-v2h97 I0330 14:59:47.042195 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-dns/node-resolver-458lx after 0 failed attempt(s) I0330 14:59:47.042199 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-p9wnr I0330 14:59:47.042160 3819 obj_retry.go:364] Adding new object: *v1.Pod kube-system/global-pull-secret-syncer-hft9h I0330 14:59:47.042207 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-vpbs4 after 0 failed attempt(s) I0330 14:59:47.042133 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/multus-4v7ww I0330 14:59:47.042224 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-p9wnr after 0 failed attempt(s) I0330 14:59:47.042231 3819 base_network_controller_pods.go:487] [default/kube-system/global-pull-secret-syncer-hft9h] creating logical port kube-system_global-pull-secret-syncer-hft9h for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:47.042197 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-image-registry/node-ca-v2h97 I0330 14:59:47.042255 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-image-registry/node-ca-v2h97 after 0 failed attempt(s) I0330 14:59:47.042147 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-sj6h6 I0330 14:59:47.042297 3819 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-sj6h6] creating logical port openshift-multus_network-metrics-daemon-sj6h6 for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:47.042131 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-z22tc I0330 14:59:47.042349 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-z22tc I0330 14:59:47.042374 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-z22tc after 0 failed attempt(s) I0330 14:59:47.042148 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-c2f7q I0330 14:59:47.042385 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-c2f7q I0330 14:59:47.042404 3819 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-c2f7q] creating logical port openshift-network-diagnostics_network-check-target-c2f7q for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:47.042162 3819 obj_retry.go:364] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-27.ec2.internal I0330 14:59:47.042511 3819 obj_retry.go:384] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-1-27.ec2.internal after 0 failed attempt(s) I0330 14:59:47.042159 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-n9hvs I0330 14:59:47.042553 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-n9hvs I0330 14:59:47.042575 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-n9hvs after 0 failed attempt(s) I0330 14:59:47.042175 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-svxb4 I0330 14:59:47.042599 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-svxb4 after 0 failed attempt(s) I0330 14:59:47.042170 3819 obj_retry.go:375] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-mjzq2, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-mjzq2: suppressed error logged: pod openshift-multus/network-metrics-daemon-mjzq2: no pod IPs found I0330 14:59:47.042118 3819 obj_retry.go:300] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-27.ec2.internal I0330 14:59:47.042616 3819 obj_retry.go:364] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-27.ec2.internal I0330 14:59:47.042632 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-27.ec2.internal after 0 failed attempt(s) I0330 14:59:47.042232 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/multus-4v7ww after 0 failed attempt(s) I0330 14:59:47.042980 3819 pods.go:271] [kube-system/global-pull-secret-syncer-hft9h] addLogicalPort took 756.399µs, libovsdb time 313.867µs I0330 14:59:47.042992 3819 obj_retry.go:384] Retry successful for *v1.Pod kube-system/global-pull-secret-syncer-hft9h after 0 failed attempt(s) I0330 14:59:47.043070 3819 pods.go:271] [openshift-multus/network-metrics-daemon-sj6h6] addLogicalPort took 783.809µs, libovsdb time 308.451µs I0330 14:59:47.043079 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-sj6h6 after 0 failed attempt(s) I0330 14:59:47.043154 3819 pods.go:271] [openshift-network-diagnostics/network-check-target-c2f7q] addLogicalPort took 755.514µs, libovsdb time 384.474µs I0330 14:59:47.043164 3819 obj_retry.go:384] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-c2f7q after 0 failed attempt(s) I0330 14:59:47.052998 3819 gateway_shared_intf.go:1770] Creating Gateway Node Port Watcher I0330 14:59:47.064578 3819 iptables.go:108] Creating table: mangle chain: OUTPUT I0330 14:59:47.067876 3819 iptables.go:108] Creating table: nat chain: OUTPUT I0330 14:59:47.091377 3819 iptables.go:108] Creating table: nat chain: PREROUTING I0330 14:59:47.145074 3819 iptables.go:108] Creating table: filter chain: FORWARD I0330 14:59:47.162803 3819 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.1 W0330 14:59:47.162927 3819 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0330 14:59:47.162990 3819 gateway_shared_intf.go:2095] Ensuring IP Neighbor entry for: 169.254.0.4 W0330 14:59:47.163089 3819 gateway_shared_intf.go:2101] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0330 14:59:47.163147 3819 gateway_init.go:325] Gateway and management port readiness took 1.679257501s I0330 14:59:47.163175 3819 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0330 14:59:47.163212 3819 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0330 14:59:47.163224 3819 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0330 14:59:47.163233 3819 nad_controller.go:241] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0330 14:59:47.163292 3819 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0330 14:59:47.163374 3819 nad_controller.go:280] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 54.083µs I0330 14:59:47.163679 3819 network_controller.go:246] [node-nad-controller network controller]: syncing all networks I0330 14:59:47.163706 3819 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 16.107µs I0330 14:59:47.163714 3819 network_controller.go:257] [node-nad-controller network controller]: finished syncing all networks. Time taken: 36.245µs I0330 14:59:47.163719 3819 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0330 14:59:47.163729 3819 nad_controller.go:173] [node-nad-controller NAD controller]: started I0330 14:59:47.163734 3819 default_node_network_controller.go:898] Starting the default node network controller I0330 14:59:47.163738 3819 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0330 14:59:47.163811 3819 network_controller.go:275] [node-nad-controller network controller]: finished syncing network default, took 12.308µs I0330 14:59:47.163917 3819 default_node_network_controller.go:1444] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0330 14:59:47.163926 3819 default_node_network_controller.go:934] Waiting for gateway and management port readiness... I0330 14:59:47.163949 3819 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000131db0 0xc0002b5ea0 0xc0002b5f10 0xc0002ec000 0xc0002ec070 0xc0002ec0e0 0xc0002ec150 0xc000ab2c30 0xc0002ec1c0 0xc0002ec230 0xc000ab2c80 0xc006479180 0xc0002ec2a0 map[0x2f67040:0xc00279eae0 0x2f67580:0xc00279ec60 0x2f68000:0xc00250d9e0 0x2f682a0:0xc00279ea80 0x2f687e0:0xc00279eb40 0x2f68a80:0xc00279eba0 0x2f68d20:0xc00279ec00 0x2f69500:0xc00279ecc0 0x2f697a0:0xc00279ed20 0x2f93d40:0xc0009a7260 0x2f940e0:0xc0009a72c0 0x2f94bc0:0xc000ab5560 0x2f96520:0xc001bf0000 0x2f97000:0xc00250d980 0x2f97e80:0xc001118900] 0xc00016a690 0} I0330 14:59:47.164018 3819 gateway.go:151] Starting gateway service sync I0330 14:59:47.164463 3819 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0330 14:59:47.166633 3819 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0330 14:59:47.176297 3819 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0330 14:59:47.177484 3819 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0330 14:59:47.178519 3819 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0330 14:59:47.215961 3819 gateway.go:168] Gateway service sync done. Time taken: 51.928169ms I0330 14:59:47.216293 3819 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 32405 I0330 14:59:47.216821 3819 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000131db0 0xc0002b5ea0 0xc0002b5f10 0xc0002ec000 0xc0002ec070 0xc0002ec0e0 0xc0002ec150 0xc000ab2c30 0xc0002ec1c0 0xc0002ec230 0xc000ab2c80 0xc006479180 0xc0002ec2a0 map[0x2f67040:0xc00279eae0 0x2f67580:0xc00279ec60 0x2f68000:0xc00250d9e0 0x2f682a0:0xc00279ea80 0x2f687e0:0xc00279eb40 0x2f68a80:0xc00279eba0 0x2f68d20:0xc00279ec00 0x2f69500:0xc00279ecc0 0x2f697a0:0xc00279ed20 0x2f93d40:0xc0009a7260 0x2f940e0:0xc0009a72c0 0x2f94bc0:0xc000ab5560 0x2f96520:0xc001bf0000 0x2f97000:0xc00250d980 0x2f97e80:0xc001118900] 0xc00016a690 0} I0330 14:59:47.217421 3819 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000131db0 0xc0002b5ea0 0xc0002b5f10 0xc0002ec000 0xc0002ec070 0xc0002ec0e0 0xc0002ec150 0xc000ab2c30 0xc0002ec1c0 0xc0002ec230 0xc000ab2c80 0xc006479180 0xc0002ec2a0 map[0x2f67040:0xc00279eae0 0x2f67580:0xc00279ec60 0x2f68000:0xc00250d9e0 0x2f682a0:0xc00279ea80 0x2f687e0:0xc00279eb40 0x2f68a80:0xc00279eba0 0x2f68d20:0xc00279ec00 0x2f69500:0xc00279ecc0 0x2f697a0:0xc00279ed20 0x2f93d40:0xc0009a7260 0x2f940e0:0xc0009a72c0 0x2f94bc0:0xc000ab5560 0x2f96520:0xc001bf0000 0x2f97000:0xc00250d980 0x2f97e80:0xc001118900] 0xc00016a690 0} I0330 14:59:47.217465 3819 gateway.go:519] Reconciling gateway with updates I0330 14:59:47.223690 3819 gateway.go:326] Spawning Conntrack Rule Check Thread I0330 14:59:47.223774 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0330 14:59:47.223793 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0330 14:59:47.223806 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0330 14:59:47.223806 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0330 14:59:47.223806 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0330 14:59:47.223823 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0330 14:59:47.223821 3819 default_node_network_controller.go:943] Gateway and management port readiness took 59.887235ms I0330 14:59:47.223827 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0330 14:59:47.223834 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0330 14:59:47.223882 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0330 14:59:47.223891 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0330 14:59:47.223895 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0330 14:59:47.223778 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0330 14:59:47.223910 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0330 14:59:47.223914 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0330 14:59:47.223883 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0330 14:59:47.223968 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0330 14:59:47.223989 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0330 14:59:47.223990 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0330 14:59:47.223992 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0330 14:59:47.224003 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0330 14:59:47.224010 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0330 14:59:47.224014 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0330 14:59:47.224024 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0330 14:59:47.224033 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0330 14:59:47.224034 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0330 14:59:47.224038 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0330 14:59:47.224044 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0330 14:59:47.224044 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0330 14:59:47.224034 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0330 14:59:47.224050 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0330 14:59:47.224012 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0330 14:59:47.224057 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0330 14:59:47.223988 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0330 14:59:47.224003 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0330 14:59:47.224064 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0330 14:59:47.224070 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0330 14:59:47.224079 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0330 14:59:47.224082 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0330 14:59:47.224081 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0330 14:59:47.224092 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0330 14:59:47.224080 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0330 14:59:47.224093 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0330 14:59:47.224099 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0330 14:59:47.224106 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0330 14:59:47.224106 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0330 14:59:47.224019 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0330 14:59:47.224109 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0330 14:59:47.224124 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0330 14:59:47.224127 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0330 14:59:47.224066 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0330 14:59:47.224139 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0330 14:59:47.224124 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0330 14:59:47.224169 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0330 14:59:47.224171 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0330 14:59:47.224185 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0330 14:59:47.224116 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0330 14:59:47.224063 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0330 14:59:47.224157 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0330 14:59:47.224203 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0330 14:59:47.224204 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0330 14:59:47.224216 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0330 14:59:47.224046 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0330 14:59:47.224221 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0330 14:59:47.224226 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0330 14:59:47.224232 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0330 14:59:47.224231 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0330 14:59:47.224187 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0330 14:59:47.224241 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0330 14:59:47.224216 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0330 14:59:47.224276 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0330 14:59:47.224280 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0330 14:59:47.224279 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0330 14:59:47.224289 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0330 14:59:47.224292 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0330 14:59:47.224296 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0330 14:59:47.224299 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0330 14:59:47.224306 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0330 14:59:47.224241 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0330 14:59:47.224320 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0330 14:59:47.224276 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0330 14:59:47.224331 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0330 14:59:47.224248 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0330 14:59:47.224260 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0330 14:59:47.224402 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0330 14:59:47.224267 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0330 14:59:47.224416 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0330 14:59:47.224424 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0330 14:59:47.224433 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0330 14:59:47.224286 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0330 14:59:47.224449 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0330 14:59:47.224436 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0330 14:59:47.224465 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0330 14:59:47.224465 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0330 14:59:47.224479 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0330 14:59:47.224492 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0330 14:59:47.224505 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0330 14:59:47.224287 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0330 14:59:47.224511 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0330 14:59:47.224520 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0330 14:59:47.224523 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0330 14:59:47.224298 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0330 14:59:47.224312 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0330 14:59:47.224591 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0330 14:59:47.224315 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0330 14:59:47.224648 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0330 14:59:47.224339 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0330 14:59:47.224207 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0330 14:59:47.224780 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0330 14:59:47.224376 3819 port_claim.go:71] Svc openshift-ingress/router-default: port 32409 is already open, no action needed I0330 14:59:47.224850 3819 port_claim.go:71] Svc openshift-ingress/router-default: port 31618 is already open, no action needed I0330 14:59:47.225022 3819 node_ip_handler_linux.go:160] Node IP manager is running W0330 14:59:47.262971 3819 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0330 14:59:47.263161 3819 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0330 14:59:47.263326 3819 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0330 14:59:47.274184 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0330 14:59:47.274209 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0330 14:59:47.274217 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0330 14:59:47.274223 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0330 14:59:47.274230 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0330 14:59:47.274237 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0330 14:59:47.274243 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0330 14:59:47.274248 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0330 14:59:47.274292 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0330 14:59:47.274312 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0330 14:59:47.274320 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0330 14:59:47.274326 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0330 14:59:47.274340 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0330 14:59:47.274363 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0330 14:59:47.274375 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0330 14:59:47.274383 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0330 14:59:47.274390 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0330 14:59:47.313070 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0330 14:59:47.313110 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0330 14:59:47.313123 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0330 14:59:47.313132 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0330 14:59:47.313143 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0330 14:59:47.313153 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0330 14:59:47.354347 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) W0330 14:59:47.619699 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0330 14:59:47.898156 3819 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0330 14:59:47.899134 3819 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 955.064µs I0330 14:59:47.912494 3819 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0330 14:59:48.276630 3819 namespace.go:98] [openshift-service-ca] adding namespace I0330 14:59:48.277572 3819 namespace.go:102] [openshift-service-ca] adding namespace took 917.673µs I0330 14:59:48.291808 3819 namespace.go:142] [openshift-service-ca] updating namespace I0330 14:59:49.142254 3819 default_node_network_controller.go:1364] Starting node controller node sync W0330 14:59:49.142295 3819 egressip_healthcheck.go:76] Health checking using insecure connection I0330 14:59:49.142342 3819 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0330 14:59:49.154503 3819 default_node_network_controller.go:1406] Node controller node sync done. Time taken: 12.223692ms I0330 14:59:49.154649 3819 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-167.ec2.internal", IP: 10.0.2.167 to PMTUD blocking rules I0330 14:59:49.154664 3819 default_node_network_controller.go:1277] Adding remote node "ip-10-0-2-167.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0330 14:59:49.160850 3819 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0330 14:59:49.161166 3819 egressservice_node.go:93] Setting up event handlers for Egress Services I0330 14:59:49.161283 3819 egressservice_node.go:183] Starting Egress Services Controller I0330 14:59:49.161299 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0330 14:59:49.161310 3819 shared_informer.go:356] "Caches are synced" controller="egressservices" I0330 14:59:49.161318 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0330 14:59:49.161326 3819 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0330 14:59:49.161345 3819 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0330 14:59:49.161354 3819 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0330 14:59:49.161361 3819 egressservice_node.go:197] Repairing Egress Services I0330 14:59:49.172094 3819 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0330 14:59:49.172109 3819 external_controller.go:278] Starting Admin Policy Based Route Controller I0330 14:59:49.172305 3819 default_node_network_controller.go:1076] Egress IP for secondary host network is disabled I0330 14:59:49.172318 3819 link_network_manager.go:117] Link manager is running I0330 14:59:49.172332 3819 default_node_network_controller.go:1087] Default node network controller initialized and ready. I0330 14:59:49.172387 3819 ovspinning_linux.go:42] OVS CPU affinity pinning disabled I0330 14:59:49.172730 3819 vrf_manager.go:128] VRF manager is running I0330 14:59:49.172853 3819 node_controller_manager.go:437] Removing flows to drop GARP I0330 14:59:49.172872 3819 gateway.go:519] Reconciling gateway with updates I0330 14:59:49.191692 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0330 14:59:49.191721 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0330 14:59:49.191718 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0330 14:59:49.191731 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0330 14:59:49.191733 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0330 14:59:49.191743 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0330 14:59:49.191749 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0330 14:59:49.191754 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0330 14:59:49.191760 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0330 14:59:49.191755 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0330 14:59:49.191765 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0330 14:59:49.191762 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0330 14:59:49.191776 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0330 14:59:49.191777 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0330 14:59:49.191772 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0330 14:59:49.191791 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0330 14:59:49.191793 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0330 14:59:49.191797 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0330 14:59:49.191796 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0330 14:59:49.191795 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0330 14:59:49.191808 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0330 14:59:49.191810 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0330 14:59:49.191811 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/packageserver I0330 14:59:49.191814 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0330 14:59:49.191817 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0330 14:59:49.191737 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0330 14:59:49.191822 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/packageserver I0330 14:59:49.191823 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0330 14:59:49.191829 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0330 14:59:49.191833 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0330 14:59:49.191816 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0330 14:59:49.191844 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0330 14:59:49.191844 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0330 14:59:49.191850 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0330 14:59:49.191849 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0330 14:59:49.191855 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0330 14:59:49.191860 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0330 14:59:49.191859 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift I0330 14:59:49.191864 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0330 14:59:49.191865 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0330 14:59:49.191747 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0330 14:59:49.191853 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0330 14:59:49.191874 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0330 14:59:49.191877 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0330 14:59:49.191879 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0330 14:59:49.191878 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0330 14:59:49.191886 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0330 14:59:49.191884 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0330 14:59:49.191888 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0330 14:59:49.191892 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0330 14:59:49.191894 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0330 14:59:49.191897 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0330 14:59:49.191899 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0330 14:59:49.191913 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0330 14:59:49.191781 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0330 14:59:49.191890 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0330 14:59:49.191868 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0330 14:59:49.191935 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0330 14:59:49.191940 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0330 14:59:49.191943 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway default/kubernetes I0330 14:59:49.191948 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0330 14:59:49.191785 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0330 14:59:49.191957 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0330 14:59:49.191967 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0330 14:59:49.191973 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0330 14:59:49.191973 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0330 14:59:49.191978 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0330 14:59:49.191982 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0330 14:59:49.191881 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0330 14:59:49.191897 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0330 14:59:49.191817 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0330 14:59:49.191817 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0330 14:59:49.192030 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0330 14:59:49.191792 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0330 14:59:49.192055 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0330 14:59:49.192066 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0330 14:59:49.191815 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0330 14:59:49.192070 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0330 14:59:49.192084 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0330 14:59:49.192086 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0330 14:59:49.192085 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0330 14:59:49.192092 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0330 14:59:49.191827 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0330 14:59:49.192095 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0330 14:59:49.192113 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0330 14:59:49.192115 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0330 14:59:49.191823 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0330 14:59:49.192127 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0330 14:59:49.192128 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0330 14:59:49.191716 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0330 14:59:49.192134 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0330 14:59:49.191837 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0330 14:59:49.192144 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0330 14:59:49.192151 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0330 14:59:49.191834 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0330 14:59:49.192161 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0330 14:59:49.191839 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0330 14:59:49.191869 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0330 14:59:49.192199 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0330 14:59:49.191873 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift I0330 14:59:49.191872 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0330 14:59:49.192221 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0330 14:59:49.192221 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0330 14:59:49.191889 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0330 14:59:49.192231 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0330 14:59:49.192244 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0330 14:59:49.191898 3819 obj_retry.go:300] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0330 14:59:49.192256 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0330 14:59:49.192261 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0330 14:59:49.191931 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0330 14:59:49.191908 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0330 14:59:49.191947 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0330 14:59:49.192235 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0330 14:59:49.191804 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0330 14:59:49.192023 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0330 14:59:49.191858 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0330 14:59:49.191958 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/kubernetes I0330 14:59:49.191923 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0330 14:59:49.192170 3819 obj_retry.go:364] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0330 14:59:49.192419 3819 port_claim.go:71] Svc openshift-ingress/router-default: port 32409 is already open, no action needed I0330 14:59:49.192431 3819 port_claim.go:71] Svc openshift-ingress/router-default: port 31618 is already open, no action needed I0330 14:59:49.216706 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0330 14:59:49.216740 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0330 14:59:49.216750 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0330 14:59:49.216771 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0330 14:59:49.216778 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0330 14:59:49.216785 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0330 14:59:49.216791 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0330 14:59:49.216797 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0330 14:59:49.216803 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0330 14:59:49.216810 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0330 14:59:49.261594 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0330 14:59:49.261625 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0330 14:59:49.303101 3819 obj_retry.go:384] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0330 14:59:50.719249 3819 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-27.ec2.internal I0330 14:59:50.719276 3819 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-27.ec2.internal: 35.35µs I0330 14:59:50.744470 3819 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-84884f658b-pb2kz] creating logical port openshift-image-registry_image-registry-84884f658b-pb2kz for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:50.744620 3819 kube.go:256] Updating pod openshift-image-registry/image-registry-84884f658b-pb2kz I0330 14:59:50.763196 3819 pod.go:62] [openshift-image-registry/image-registry-84884f658b-pb2kz] pod update took 18.604005ms I0330 14:59:50.763225 3819 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-84884f658b-pb2kz] addLogicalPort annotation time took 18.635922ms I0330 14:59:50.764608 3819 pods.go:271] [openshift-image-registry/image-registry-84884f658b-pb2kz] addLogicalPort took 20.155425ms, libovsdb time 1.029045ms I0330 14:59:50.771861 3819 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-78459bd87c-rz9gd] creating logical port openshift-ingress_router-default-78459bd87c-rz9gd for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:50.772052 3819 kube.go:256] Updating pod openshift-ingress/router-default-78459bd87c-rz9gd I0330 14:59:50.783148 3819 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-2fs4z] creating logical port openshift-dns_dns-default-2fs4z for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:50.783195 3819 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-2m285] creating logical port openshift-ingress-canary_ingress-canary-2m285 for pod on switch ip-10-0-1-27.ec2.internal I0330 14:59:50.783267 3819 kube.go:256] Updating pod openshift-dns/dns-default-2fs4z I0330 14:59:50.783318 3819 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-2m285 I0330 14:59:50.802729 3819 pod.go:62] [openshift-ingress/router-default-78459bd87c-rz9gd] pod update took 30.72153ms I0330 14:59:50.802778 3819 base_network_controller_pods.go:937] [default/openshift-ingress/router-default-78459bd87c-rz9gd] addLogicalPort annotation time took 30.749705ms I0330 14:59:50.803922 3819 pod.go:62] [openshift-dns/dns-default-2fs4z] pod update took 20.698729ms I0330 14:59:50.803940 3819 base_network_controller_pods.go:937] [default/openshift-dns/dns-default-2fs4z] addLogicalPort annotation time took 20.723169ms I0330 14:59:50.804247 3819 pods.go:271] [openshift-ingress/router-default-78459bd87c-rz9gd] addLogicalPort took 32.404505ms, libovsdb time 1.154166ms I0330 14:59:50.805250 3819 pods.go:271] [openshift-dns/dns-default-2fs4z] addLogicalPort took 22.12257ms, libovsdb time 762.538µs I0330 14:59:50.806190 3819 pod.go:62] [openshift-ingress-canary/ingress-canary-2m285] pod update took 22.909447ms I0330 14:59:50.806210 3819 base_network_controller_pods.go:937] [default/openshift-ingress-canary/ingress-canary-2m285] addLogicalPort annotation time took 22.930428ms I0330 14:59:50.807130 3819 pods.go:271] [openshift-ingress-canary/ingress-canary-2m285] addLogicalPort took 23.950354ms, libovsdb time 700.211µs I0330 14:59:51.152968 3819 cni.go:403] [openshift-ingress-canary/ingress-canary-2m285 fce35406b0bc67783ae64bc09ad179d429a076c9a1006b27bb628779568c8599 network default NAD default] ADD starting CNI request [openshift-ingress-canary/ingress-canary-2m285 fce35406b0bc67783ae64bc09ad179d429a076c9a1006b27bb628779568c8599 network default NAD default] I0330 14:59:51.174945 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-2m285, hostIfaceName: fce35406b0bc677, network: default, NAD default, SandboxID: "fce35406b0bc67783ae64bc09ad179d429a076c9a1006b27bb628779568c8599", PCI device ID: , UID: "ecb59fd8-facb-45cc-8c85-bdcf904b52d6", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0330 14:59:51.228560 3819 cni.go:424] [openshift-ingress-canary/ingress-canary-2m285 fce35406b0bc67783ae64bc09ad179d429a076c9a1006b27bb628779568c8599 network default NAD default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-2m285 fce35406b0bc67783ae64bc09ad179d429a076c9a1006b27bb628779568c8599 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:d2:ef:d6:d6:49\",\"name\":\"fce35406b0bc677\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/80f46321-e54c-472a-89cf-e6565b106023\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 14:59:53.847368 3819 cni.go:403] [openshift-network-diagnostics/network-check-target-c2f7q 653117d3b35508a0187225fc0cbdde4b4c2f89b2394ffb525b38ff30d940592b network default NAD default] ADD starting CNI request [openshift-network-diagnostics/network-check-target-c2f7q 653117d3b35508a0187225fc0cbdde4b4c2f89b2394ffb525b38ff30d940592b network default NAD default] I0330 14:59:53.856326 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-c2f7q, hostIfaceName: 653117d3b35508a, network: default, NAD default, SandboxID: "653117d3b35508a0187225fc0cbdde4b4c2f89b2394ffb525b38ff30d940592b", PCI device ID: , UID: "bca34613-ce54-4994-bc95-2d4c0eac6489", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0330 14:59:53.899858 3819 cni.go:424] [openshift-network-diagnostics/network-check-target-c2f7q 653117d3b35508a0187225fc0cbdde4b4c2f89b2394ffb525b38ff30d940592b network default NAD default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-c2f7q 653117d3b35508a0187225fc0cbdde4b4c2f89b2394ffb525b38ff30d940592b network default NAD default], result "{\"interfaces\":[{\"mac\":\"1a:8b:db:aa:39:fa\",\"name\":\"653117d3b35508a\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/52593e5f-6b3f-44d3-b0d9-d023c5849687\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 14:59:55.592843 3819 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-84884f658b-w459r IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-84884f658b-w459r: no pod IPs found W0330 14:59:55.742976 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0330 14:59:56.659713 3819 obj_retry.go:464] Detected object openshift-image-registry/image-registry-8547d789cc-z8sdp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 14:59:56.660467 3819 obj_retry.go:464] Detected object openshift-image-registry/image-registry-8547d789cc-z8sdp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 14:59:56.677377 3819 obj_retry.go:458] Detected object openshift-image-registry/image-registry-8547d789cc-z8sdp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 14:59:56.677405 3819 obj_retry.go:458] Detected object openshift-image-registry/image-registry-8547d789cc-z8sdp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 14:59:56.685378 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-8547d789cc-z8sdp I0330 14:59:56.685408 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-8547d789cc-z8sdp I0330 14:59:58.617876 3819 cni.go:403] [openshift-image-registry/image-registry-84884f658b-pb2kz 68283857f83395ef6667407a63e485429dae0c050d93355b426a9a10236c16be network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-84884f658b-pb2kz 68283857f83395ef6667407a63e485429dae0c050d93355b426a9a10236c16be network default NAD default] I0330 14:59:58.627101 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-84884f658b-pb2kz, hostIfaceName: 68283857f83395e, network: default, NAD default, SandboxID: "68283857f83395ef6667407a63e485429dae0c050d93355b426a9a10236c16be", PCI device ID: , UID: "5f9bc1bd-7ba0-4122-9b4b-2e79c4e63c6f", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0330 14:59:58.669648 3819 cni.go:424] [openshift-image-registry/image-registry-84884f658b-pb2kz 68283857f83395ef6667407a63e485429dae0c050d93355b426a9a10236c16be network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-84884f658b-pb2kz 68283857f83395ef6667407a63e485429dae0c050d93355b426a9a10236c16be network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:5d:9b:c6:d3:b1\",\"name\":\"68283857f83395e\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c9f9a4c9-0f1a-496d-aa91-a6d2168476ec\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 14:59:58.963229 3819 cni.go:403] [openshift-dns/dns-default-2fs4z 2e54411e6e271ce4fb224444b1ab6dc996abb06118f34223ff07890c2ef4e3a4 network default NAD default] ADD starting CNI request [openshift-dns/dns-default-2fs4z 2e54411e6e271ce4fb224444b1ab6dc996abb06118f34223ff07890c2ef4e3a4 network default NAD default] I0330 14:59:58.973097 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-dns, podName: dns-default-2fs4z, hostIfaceName: 2e54411e6e271ce, network: default, NAD default, SandboxID: "2e54411e6e271ce4fb224444b1ab6dc996abb06118f34223ff07890c2ef4e3a4", PCI device ID: , UID: "47448770-3e93-4d67-803d-b721d9ae61d7", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0330 14:59:59.015306 3819 cni.go:424] [openshift-dns/dns-default-2fs4z 2e54411e6e271ce4fb224444b1ab6dc996abb06118f34223ff07890c2ef4e3a4 network default NAD default] ADD finished CNI request [openshift-dns/dns-default-2fs4z 2e54411e6e271ce4fb224444b1ab6dc996abb06118f34223ff07890c2ef4e3a4 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ea:3a:a9:52:96:da\",\"name\":\"2e54411e6e271ce\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/30bd363e-63e2-4e16-9afb-b7d016d85bb9\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:00:03.436422 3819 default_network_controller.go:1008] Node "ip-10-0-0-65.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0330 15:00:04.972369 3819 cni.go:403] [kube-system/global-pull-secret-syncer-hft9h 1646bbf241c3ae2b371e1e584379a9265ba0f173408ba4071bd0596fe9ec8988 network default NAD default] ADD starting CNI request [kube-system/global-pull-secret-syncer-hft9h 1646bbf241c3ae2b371e1e584379a9265ba0f173408ba4071bd0596fe9ec8988 network default NAD default] I0330 15:00:04.981043 3819 helper_linux.go:450] ConfigureOVS: namespace: kube-system, podName: global-pull-secret-syncer-hft9h, hostIfaceName: 1646bbf241c3ae2, network: default, NAD default, SandboxID: "1646bbf241c3ae2b371e1e584379a9265ba0f173408ba4071bd0596fe9ec8988", PCI device ID: , UID: "7f4939f9-f7e4-4ae7-8b13-a4e4c36c4ee9", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0330 15:00:05.023890 3819 cni.go:424] [kube-system/global-pull-secret-syncer-hft9h 1646bbf241c3ae2b371e1e584379a9265ba0f173408ba4071bd0596fe9ec8988 network default NAD default] ADD finished CNI request [kube-system/global-pull-secret-syncer-hft9h 1646bbf241c3ae2b371e1e584379a9265ba0f173408ba4071bd0596fe9ec8988 network default NAD default], result "{\"interfaces\":[{\"mac\":\"e2:25:79:10:12:6b\",\"name\":\"1646bbf241c3ae2\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/06a62e19-9b87-448a-a448-1750c8a293ba\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:00:06.762688 3819 cni.go:403] [openshift-ingress/router-default-78459bd87c-rz9gd 0b401b53f06baad3b8f16f00d6a50d243012c7ecc3d70967a4bcc2dbb8c916f0 network default NAD default] ADD starting CNI request [openshift-ingress/router-default-78459bd87c-rz9gd 0b401b53f06baad3b8f16f00d6a50d243012c7ecc3d70967a4bcc2dbb8c916f0 network default NAD default] I0330 15:00:06.772511 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-ingress, podName: router-default-78459bd87c-rz9gd, hostIfaceName: 0b401b53f06baad, network: default, NAD default, SandboxID: "0b401b53f06baad3b8f16f00d6a50d243012c7ecc3d70967a4bcc2dbb8c916f0", PCI device ID: , UID: "4db81e1c-1fdf-41ba-99dd-30284d2bcf6f", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0330 15:00:06.815830 3819 cni.go:424] [openshift-ingress/router-default-78459bd87c-rz9gd 0b401b53f06baad3b8f16f00d6a50d243012c7ecc3d70967a4bcc2dbb8c916f0 network default NAD default] ADD finished CNI request [openshift-ingress/router-default-78459bd87c-rz9gd 0b401b53f06baad3b8f16f00d6a50d243012c7ecc3d70967a4bcc2dbb8c916f0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:22:4a:7e:89:7e\",\"name\":\"0b401b53f06baad\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/08ff25c3-80ca-435b-9d06-e55676eff154\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:00:14.919070 3819 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0330 15:00:14.932201 3819 default_network_controller.go:1008] Node "ip-10-0-0-65.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0330 15:00:14.983458 3819 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0330 15:00:15.003511 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0330 15:00:15.026611 3819 default_network_controller.go:1008] Node "ip-10-0-0-65.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0330 15:00:15.027113 3819 base_network_controller_pods.go:487] [default/openshift-console/console-6c6664555d-w9m8v] creating logical port openshift-console_console-6c6664555d-w9m8v for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:15.027280 3819 kube.go:256] Updating pod openshift-console/console-6c6664555d-w9m8v I0330 15:00:15.073797 3819 pod.go:62] [openshift-console/console-6c6664555d-w9m8v] pod update took 46.562512ms I0330 15:00:15.073823 3819 base_network_controller_pods.go:937] [default/openshift-console/console-6c6664555d-w9m8v] addLogicalPort annotation time took 46.594078ms I0330 15:00:15.074935 3819 pods.go:271] [openshift-console/console-6c6664555d-w9m8v] addLogicalPort took 47.839147ms, libovsdb time 843.638µs I0330 15:00:15.075004 3819 base_network_controller_pods.go:487] [default/openshift-console/downloads-56d45c498b-lvhww] creating logical port openshift-console_downloads-56d45c498b-lvhww for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:15.075079 3819 kube.go:256] Updating pod openshift-console/downloads-56d45c498b-lvhww I0330 15:00:15.112449 3819 pod.go:62] [openshift-console/downloads-56d45c498b-lvhww] pod update took 37.396037ms I0330 15:00:15.112477 3819 base_network_controller_pods.go:937] [default/openshift-console/downloads-56d45c498b-lvhww] addLogicalPort annotation time took 37.431848ms I0330 15:00:15.113649 3819 pods.go:271] [openshift-console/downloads-56d45c498b-lvhww] addLogicalPort took 38.650372ms, libovsdb time 811.401µs I0330 15:00:15.143959 3819 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-n4wnp] creating logical port openshift-insights_insights-runtime-extractor-n4wnp for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:15.144081 3819 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-n4wnp W0330 15:00:15.149955 3819 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0330 15:00:15.198729 3819 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29581380-ssrkd for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:15.198813 3819 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd I0330 15:00:15.214844 3819 pod.go:62] [openshift-insights/insights-runtime-extractor-n4wnp] pod update took 70.795509ms I0330 15:00:15.214873 3819 base_network_controller_pods.go:937] [default/openshift-insights/insights-runtime-extractor-n4wnp] addLogicalPort annotation time took 70.83222ms I0330 15:00:15.216005 3819 pods.go:271] [openshift-insights/insights-runtime-extractor-n4wnp] addLogicalPort took 72.064682ms, libovsdb time 773.529µs I0330 15:00:15.228187 3819 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd] pod update took 29.392212ms I0330 15:00:15.228214 3819 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd] addLogicalPort annotation time took 29.424795ms I0330 15:00:15.229336 3819 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd] addLogicalPort took 30.619659ms, libovsdb time 815.685µs W0330 15:00:15.229412 3819 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-image-registry/image-registry-7967d4f498-mql52 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-7967d4f498-mql52: no pod IPs found I0330 15:00:15.423728 3819 cni.go:403] [openshift-console/console-6c6664555d-w9m8v 8ad73945d6ca1ffdb97fc04fdbaa528abdd64ad7d4fe4e391b01379c9b11c9ab network default NAD default] ADD starting CNI request [openshift-console/console-6c6664555d-w9m8v 8ad73945d6ca1ffdb97fc04fdbaa528abdd64ad7d4fe4e391b01379c9b11c9ab network default NAD default] I0330 15:00:15.432757 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-6c6664555d-w9m8v, hostIfaceName: 8ad73945d6ca1ff, network: default, NAD default, SandboxID: "8ad73945d6ca1ffdb97fc04fdbaa528abdd64ad7d4fe4e391b01379c9b11c9ab", PCI device ID: , UID: "f255768b-f1fb-4ea7-9f58-13e0b6dbc32b", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0330 15:00:15.452035 3819 cni.go:403] [openshift-console/downloads-56d45c498b-lvhww cf08115a22d7104314145a439bc5329eacf5836b2d5fa989126bf184514a8c5f network default NAD default] ADD starting CNI request [openshift-console/downloads-56d45c498b-lvhww cf08115a22d7104314145a439bc5329eacf5836b2d5fa989126bf184514a8c5f network default NAD default] I0330 15:00:15.461767 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: downloads-56d45c498b-lvhww, hostIfaceName: cf08115a22d7104, network: default, NAD default, SandboxID: "cf08115a22d7104314145a439bc5329eacf5836b2d5fa989126bf184514a8c5f", PCI device ID: , UID: "a0ba918f-4715-4c53-9b20-d6e4c6bce545", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0330 15:00:15.479891 3819 cni.go:424] [openshift-console/console-6c6664555d-w9m8v 8ad73945d6ca1ffdb97fc04fdbaa528abdd64ad7d4fe4e391b01379c9b11c9ab network default NAD default] ADD finished CNI request [openshift-console/console-6c6664555d-w9m8v 8ad73945d6ca1ffdb97fc04fdbaa528abdd64ad7d4fe4e391b01379c9b11c9ab network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:2d:a0:41:c6:e3\",\"name\":\"8ad73945d6ca1ff\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cd3e6c84-d51d-4c2d-a504-0f73dc099cdf\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:00:15.505024 3819 cni.go:424] [openshift-console/downloads-56d45c498b-lvhww cf08115a22d7104314145a439bc5329eacf5836b2d5fa989126bf184514a8c5f network default NAD default] ADD finished CNI request [openshift-console/downloads-56d45c498b-lvhww cf08115a22d7104314145a439bc5329eacf5836b2d5fa989126bf184514a8c5f network default NAD default], result "{\"interfaces\":[{\"mac\":\"22:9f:4c:f8:d5:25\",\"name\":\"cf08115a22d7104\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2d933519-a417-4996-90b7-113e0ef8945c\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:00:15.558697 3819 cni.go:403] [openshift-insights/insights-runtime-extractor-n4wnp ee0313a28a37db3272bfff0cde520aa23b91f27b464bfaee0f6ee79a04266189 network default NAD default] ADD starting CNI request [openshift-insights/insights-runtime-extractor-n4wnp ee0313a28a37db3272bfff0cde520aa23b91f27b464bfaee0f6ee79a04266189 network default NAD default] I0330 15:00:15.568471 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-n4wnp, hostIfaceName: ee0313a28a37db3, network: default, NAD default, SandboxID: "ee0313a28a37db3272bfff0cde520aa23b91f27b464bfaee0f6ee79a04266189", PCI device ID: , UID: "c4705cfb-39fd-43ee-8d07-af719c7eb41d", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0330 15:00:15.615345 3819 cni.go:424] [openshift-insights/insights-runtime-extractor-n4wnp ee0313a28a37db3272bfff0cde520aa23b91f27b464bfaee0f6ee79a04266189 network default NAD default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-n4wnp ee0313a28a37db3272bfff0cde520aa23b91f27b464bfaee0f6ee79a04266189 network default NAD default], result "{\"interfaces\":[{\"mac\":\"1e:16:f7:36:60:fe\",\"name\":\"ee0313a28a37db3\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b9c62179-e6dc-4738-ab30-ca240ddf9f4b\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:00:15.631789 3819 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd 769bed655bb450a4bbf4e0aa8e89ed0cad07e4210f5038460040c2eee1c89d02 network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd 769bed655bb450a4bbf4e0aa8e89ed0cad07e4210f5038460040c2eee1c89d02 network default NAD default] I0330 15:00:15.648775 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29581380-ssrkd, hostIfaceName: 769bed655bb450a, network: default, NAD default, SandboxID: "769bed655bb450a4bbf4e0aa8e89ed0cad07e4210f5038460040c2eee1c89d02", PCI device ID: , UID: "ebe50a19-30ff-40ed-80f1-5d43ad84e82b", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0330 15:00:15.694764 3819 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd 769bed655bb450a4bbf4e0aa8e89ed0cad07e4210f5038460040c2eee1c89d02 network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd 769bed655bb450a4bbf4e0aa8e89ed0cad07e4210f5038460040c2eee1c89d02 network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:a3:b6:e8:e7:92\",\"name\":\"769bed655bb450a\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d766e654-6d9a-4b85-b682-1466ed7e032a\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:00:16.085127 3819 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-65.ec2.internal in network "default" I0330 15:00:16.085163 3819 default_network_controller.go:1008] Node "ip-10-0-0-65.ec2.internal" in remote zone "ip-10-0-0-65.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0330 15:00:16.085179 3819 node_tracker.go:235] Node ip-10-0-0-65.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-65.ec2.internal" I0330 15:00:16.085197 3819 node_tracker.go:169] Node ip-10-0-0-65.ec2.internal switch + router changed, syncing services in network "default" I0330 15:00:16.085211 3819 services_controller.go:574] Full service sync requested for network=default W0330 15:00:16.085824 3819 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0330 15:00:16.191613 3819 default_network_controller.go:1008] Node "ip-10-0-0-65.ec2.internal" in remote zone "ip-10-0-0-65.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0330 15:00:16.191684 3819 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-65.ec2.internal in network "default" I0330 15:00:16.191733 3819 node_tracker.go:169] Node ip-10-0-0-65.ec2.internal switch + router changed, syncing services in network "default" I0330 15:00:16.191748 3819 services_controller.go:574] Full service sync requested for network=default I0330 15:00:16.192284 3819 zone_ic_handler.go:260] Creating interconnect resources for remote zone node ip-10-0-0-65.ec2.internal for the network default W0330 15:00:16.192398 3819 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0330 15:00:16.193331 3819 zone_ic_handler.go:265] Creating Interconnect resources for node "ip-10-0-0-65.ec2.internal" on network "default" took: 1.060278ms I0330 15:00:16.741687 3819 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-65.ec2.internal in network "default" I0330 15:00:16.741751 3819 egressfirewall.go:628] Syncing node "ip-10-0-0-65.ec2.internal" for egress firewall I0330 15:00:16.741751 3819 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-65.ec2.internal I0330 15:00:16.741764 3819 node_tracker.go:169] Node ip-10-0-0-65.ec2.internal switch + router changed, syncing services in network "default" I0330 15:00:16.741775 3819 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-65.ec2.internal: 26.951µs I0330 15:00:16.741782 3819 services_controller.go:574] Full service sync requested for network=default I0330 15:00:16.741825 3819 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-65.ec2.internal", IP: 10.0.0.65 to PMTUD blocking rules I0330 15:00:16.741842 3819 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-65.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0330 15:00:16.742595 3819 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0330 15:00:16.748843 3819 default_network_controller.go:1067] Egress IP detected IP address change for node ip-10-0-0-65.ec2.internal. Updating no re-route policies I0330 15:00:16.761028 3819 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-65.ec2.internal", IP: 10.0.0.65 to PMTUD blocking rules I0330 15:00:16.761052 3819 default_node_network_controller.go:1277] Adding remote node "ip-10-0-0-65.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0330 15:00:17.349878 3819 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-t8mzk] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-9698cf77c-t8mzk for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:17.350015 3819 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-t8mzk I0330 15:00:17.364590 3819 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-t8mzk] pod update took 14.60084ms I0330 15:00:17.364618 3819 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-t8mzk] addLogicalPort annotation time took 14.636308ms I0330 15:00:17.366077 3819 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-t8mzk] addLogicalPort took 16.219307ms, libovsdb time 1.064999ms W0330 15:00:18.682146 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0330 15:00:18.682177 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0330 15:00:19.334502 3819 cni.go:403] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-t8mzk db6d173a8db3d8c3028e7941b3339715f2d5aa7565dbfdca80eb7c50a79a1b19 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-t8mzk db6d173a8db3d8c3028e7941b3339715f2d5aa7565dbfdca80eb7c50a79a1b19 network default NAD default] I0330 15:00:19.343645 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-9698cf77c-t8mzk, hostIfaceName: db6d173a8db3d8c, network: default, NAD default, SandboxID: "db6d173a8db3d8c3028e7941b3339715f2d5aa7565dbfdca80eb7c50a79a1b19", PCI device ID: , UID: "e1dbc271-ed1f-465f-b51a-e6df78a63960", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0330 15:00:19.387233 3819 cni.go:424] [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-t8mzk db6d173a8db3d8c3028e7941b3339715f2d5aa7565dbfdca80eb7c50a79a1b19 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-9698cf77c-t8mzk db6d173a8db3d8c3028e7941b3339715f2d5aa7565dbfdca80eb7c50a79a1b19 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:78:98:b1:16:82\",\"name\":\"db6d173a8db3d8c\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0559b8a-635d-4ce7-bf64-ffdbcd930819\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:00:19.779873 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0330 15:00:20.226794 3819 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-65.ec2.internal I0330 15:00:20.226827 3819 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-65.ec2.internal: 39.543µs W0330 15:00:20.229127 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0330 15:00:20.538901 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0330 15:00:21.810283 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0330 15:00:22.234832 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0330 15:00:22.258297 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0330 15:00:25.990471 3819 cni.go:403] [openshift-multus/network-metrics-daemon-sj6h6 5cca550a072cccb9a739523e8c115d60f0e3e770eecdb9e19b45bd22018233f7 network default NAD default] ADD starting CNI request [openshift-multus/network-metrics-daemon-sj6h6 5cca550a072cccb9a739523e8c115d60f0e3e770eecdb9e19b45bd22018233f7 network default NAD default] I0330 15:00:25.999426 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-sj6h6, hostIfaceName: 5cca550a072cccb, network: default, NAD default, SandboxID: "5cca550a072cccb9a739523e8c115d60f0e3e770eecdb9e19b45bd22018233f7", PCI device ID: , UID: "cd18b68c-f993-462c-a772-968a465b196a", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0330 15:00:26.046362 3819 cni.go:424] [openshift-multus/network-metrics-daemon-sj6h6 5cca550a072cccb9a739523e8c115d60f0e3e770eecdb9e19b45bd22018233f7 network default NAD default] ADD finished CNI request [openshift-multus/network-metrics-daemon-sj6h6 5cca550a072cccb9a739523e8c115d60f0e3e770eecdb9e19b45bd22018233f7 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:88:b9:97:57:19\",\"name\":\"5cca550a072cccb\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2111070c-3cee-4de1-bdd1-ee1ba6a5d409\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:00:26.263229 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0330 15:00:29.145382 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0330 15:00:32.120548 3819 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-1] creating logical port openshift-monitoring_alertmanager-main-1 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:32.120664 3819 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-1 I0330 15:00:32.148477 3819 pod.go:62] [openshift-monitoring/alertmanager-main-1] pod update took 27.84519ms I0330 15:00:32.148511 3819 base_network_controller_pods.go:937] [default/openshift-monitoring/alertmanager-main-1] addLogicalPort annotation time took 27.885976ms I0330 15:00:32.149910 3819 pods.go:271] [openshift-monitoring/alertmanager-main-1] addLogicalPort took 29.395301ms, libovsdb time 979.106µs I0330 15:00:32.973226 3819 cni.go:403] [openshift-monitoring/alertmanager-main-1 4623927c583b7d9fccedfcf438bf04b34b0cfb71de13489862d7f955c36b8a6f network default NAD default] ADD starting CNI request [openshift-monitoring/alertmanager-main-1 4623927c583b7d9fccedfcf438bf04b34b0cfb71de13489862d7f955c36b8a6f network default NAD default] I0330 15:00:32.982429 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-1, hostIfaceName: 4623927c583b7d9, network: default, NAD default, SandboxID: "4623927c583b7d9fccedfcf438bf04b34b0cfb71de13489862d7f955c36b8a6f", PCI device ID: , UID: "0e3c4c8f-1095-4874-93d7-b8ce2aee453d", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0330 15:00:33.028863 3819 cni.go:424] [openshift-monitoring/alertmanager-main-1 4623927c583b7d9fccedfcf438bf04b34b0cfb71de13489862d7f955c36b8a6f network default NAD default] ADD finished CNI request [openshift-monitoring/alertmanager-main-1 4623927c583b7d9fccedfcf438bf04b34b0cfb71de13489862d7f955c36b8a6f network default NAD default], result "{\"interfaces\":[{\"mac\":\"9e:53:5b:25:3f:d7\",\"name\":\"4623927c583b7d9\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bc10b432-3705-497f-ab10-283b2c597791\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:00:33.289824 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0330 15:00:33.848647 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0330 15:00:33.870237 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0330 15:00:33.870265 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0330 15:00:34.051186 3819 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-58c4c75c77-tllsn] creating logical port openshift-monitoring_thanos-querier-58c4c75c77-tllsn for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:34.051337 3819 kube.go:256] Updating pod openshift-monitoring/thanos-querier-58c4c75c77-tllsn I0330 15:00:34.074972 3819 pod.go:62] [openshift-monitoring/thanos-querier-58c4c75c77-tllsn] pod update took 23.670789ms I0330 15:00:34.075013 3819 base_network_controller_pods.go:937] [default/openshift-monitoring/thanos-querier-58c4c75c77-tllsn] addLogicalPort annotation time took 23.716373ms I0330 15:00:34.076514 3819 pods.go:271] [openshift-monitoring/thanos-querier-58c4c75c77-tllsn] addLogicalPort took 25.355078ms, libovsdb time 1.123302ms W0330 15:00:34.320575 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads I0330 15:00:34.490969 3819 cni.go:403] [openshift-monitoring/thanos-querier-58c4c75c77-tllsn dbeb8889c70db02a19b8c890a36847f497f5c1d0a39f6512542f7676a03844e1 network default NAD default] ADD starting CNI request [openshift-monitoring/thanos-querier-58c4c75c77-tllsn dbeb8889c70db02a19b8c890a36847f497f5c1d0a39f6512542f7676a03844e1 network default NAD default] I0330 15:00:34.501976 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-58c4c75c77-tllsn, hostIfaceName: dbeb8889c70db02, network: default, NAD default, SandboxID: "dbeb8889c70db02a19b8c890a36847f497f5c1d0a39f6512542f7676a03844e1", PCI device ID: , UID: "63e76a1e-286c-41aa-8bf6-42f668a7dc78", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0330 15:00:34.556413 3819 cni.go:424] [openshift-monitoring/thanos-querier-58c4c75c77-tllsn dbeb8889c70db02a19b8c890a36847f497f5c1d0a39f6512542f7676a03844e1 network default NAD default] ADD finished CNI request [openshift-monitoring/thanos-querier-58c4c75c77-tllsn dbeb8889c70db02a19b8c890a36847f497f5c1d0a39f6512542f7676a03844e1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"ca:37:a3:3a:84:93\",\"name\":\"dbeb8889c70db02\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d2cd0b7d-aa32-459d-8a6a-ee4a42021d53\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:00:34.652944 3819 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:00:34.652973 3819 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:00:34.652981 3819 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:00:35.317273 3819 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:00:35.317292 3819 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:00:35.317298 3819 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0330 15:00:35.497265 3819 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-5fd69d95b7-rbpds] creating logical port openshift-monitoring_metrics-server-5fd69d95b7-rbpds for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:35.497402 3819 kube.go:256] Updating pod openshift-monitoring/metrics-server-5fd69d95b7-rbpds I0330 15:00:35.517057 3819 pod.go:62] [openshift-monitoring/metrics-server-5fd69d95b7-rbpds] pod update took 19.695733ms I0330 15:00:35.517091 3819 base_network_controller_pods.go:937] [default/openshift-monitoring/metrics-server-5fd69d95b7-rbpds] addLogicalPort annotation time took 19.737694ms I0330 15:00:35.518642 3819 pods.go:271] [openshift-monitoring/metrics-server-5fd69d95b7-rbpds] addLogicalPort took 21.401347ms, libovsdb time 1.133664ms W0330 15:00:35.673905 3819 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-7db448cdd6-vq8vz IP address from the namespace address-set, err: pod openshift-console/console-7db448cdd6-vq8vz: no pod IPs found I0330 15:00:35.680066 3819 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-ppkjm] creating logical port openshift-monitoring_monitoring-plugin-66fd6dc9f9-ppkjm for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:35.680163 3819 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-66fd6dc9f9-ppkjm I0330 15:00:35.696063 3819 pod.go:62] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-ppkjm] pod update took 15.922852ms I0330 15:00:35.696091 3819 base_network_controller_pods.go:937] [default/openshift-monitoring/monitoring-plugin-66fd6dc9f9-ppkjm] addLogicalPort annotation time took 15.955141ms I0330 15:00:35.697326 3819 pods.go:271] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-ppkjm] addLogicalPort took 17.279245ms, libovsdb time 878.332µs I0330 15:00:35.865438 3819 cni.go:403] [openshift-monitoring/metrics-server-5fd69d95b7-rbpds 9335333e01b7065f1f9a86ed7fb691b0be51adf53ebe00935500d09564642822 network default NAD default] ADD starting CNI request [openshift-monitoring/metrics-server-5fd69d95b7-rbpds 9335333e01b7065f1f9a86ed7fb691b0be51adf53ebe00935500d09564642822 network default NAD default] I0330 15:00:35.875919 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-5fd69d95b7-rbpds, hostIfaceName: 9335333e01b7065, network: default, NAD default, SandboxID: "9335333e01b7065f1f9a86ed7fb691b0be51adf53ebe00935500d09564642822", PCI device ID: , UID: "c5f453d7-e6eb-4ffb-b9ae-51bac085e1b9", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0330 15:00:35.933511 3819 cni.go:424] [openshift-monitoring/metrics-server-5fd69d95b7-rbpds 9335333e01b7065f1f9a86ed7fb691b0be51adf53ebe00935500d09564642822 network default NAD default] ADD finished CNI request [openshift-monitoring/metrics-server-5fd69d95b7-rbpds 9335333e01b7065f1f9a86ed7fb691b0be51adf53ebe00935500d09564642822 network default NAD default], result "{\"interfaces\":[{\"mac\":\"92:3c:f7:3a:d9:cf\",\"name\":\"9335333e01b7065\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ef7e3dd4-fcb6-4f62-999e-f93f5e0dbee6\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:00:37.165573 3819 cni.go:403] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-ppkjm 29f7ab1ebc3d211d22915e9b49a66184a89611656b7b3fa30bd4ee1c21217835 network default NAD default] ADD starting CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-ppkjm 29f7ab1ebc3d211d22915e9b49a66184a89611656b7b3fa30bd4ee1c21217835 network default NAD default] I0330 15:00:37.179967 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-66fd6dc9f9-ppkjm, hostIfaceName: 29f7ab1ebc3d211, network: default, NAD default, SandboxID: "29f7ab1ebc3d211d22915e9b49a66184a89611656b7b3fa30bd4ee1c21217835", PCI device ID: , UID: "e83c2041-f2e7-4036-8308-96aa2f08427c", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0330 15:00:37.248436 3819 cni.go:424] [openshift-monitoring/monitoring-plugin-66fd6dc9f9-ppkjm 29f7ab1ebc3d211d22915e9b49a66184a89611656b7b3fa30bd4ee1c21217835 network default NAD default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-66fd6dc9f9-ppkjm 29f7ab1ebc3d211d22915e9b49a66184a89611656b7b3fa30bd4ee1c21217835 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:39:d4:d8:18:97\",\"name\":\"29f7ab1ebc3d211\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2af62625-1de2-4a18-b254-00c104cae4a5\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:00:37.327265 3819 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:37.327513 3819 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0330 15:00:37.352206 3819 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 24.753114ms I0330 15:00:37.352229 3819 base_network_controller_pods.go:937] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 24.78303ms I0330 15:00:37.353690 3819 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 26.452336ms, libovsdb time 1.107142ms I0330 15:00:38.055064 3819 cni.go:403] [openshift-monitoring/prometheus-k8s-0 92dec322ffb7c0bc75336cf90e5126085a015299f23a2d4e46b219a780935780 network default NAD default] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 92dec322ffb7c0bc75336cf90e5126085a015299f23a2d4e46b219a780935780 network default NAD default] I0330 15:00:38.064420 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 92dec322ffb7c0b, network: default, NAD default, SandboxID: "92dec322ffb7c0bc75336cf90e5126085a015299f23a2d4e46b219a780935780", PCI device ID: , UID: "1a61262f-62af-46be-a6eb-62199aacbabd", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0330 15:00:38.117814 3819 cni.go:424] [openshift-monitoring/prometheus-k8s-0 92dec322ffb7c0bc75336cf90e5126085a015299f23a2d4e46b219a780935780 network default NAD default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 92dec322ffb7c0bc75336cf90e5126085a015299f23a2d4e46b219a780935780 network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:56:da:f4:c7:dd\",\"name\":\"92dec322ffb7c0b\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a5fa8c7a-5c8f-4606-a9ab-c2174fe4e921\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:00:38.340262 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0330 15:00:38.648244 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0330 15:00:38.695681 3819 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0330 15:00:38.695703 3819 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0330 15:00:39.355344 3819 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:39.355371 3819 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:39.355378 3819 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:39.355382 3819 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:39.471176 3819 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0330 15:00:39.471196 3819 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0330 15:00:39.676677 3819 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:39.676700 3819 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:39.676709 3819 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:39.676715 3819 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0330 15:00:40.315277 3819 cni.go:403] [openshift-image-registry/image-registry-84884f658b-pb2kz 68283857f83395ef6667407a63e485429dae0c050d93355b426a9a10236c16be network default NAD default] DEL starting CNI request [openshift-image-registry/image-registry-84884f658b-pb2kz 68283857f83395ef6667407a63e485429dae0c050d93355b426a9a10236c16be network default NAD default] I0330 15:00:40.392420 3819 cni.go:424] [openshift-image-registry/image-registry-84884f658b-pb2kz 68283857f83395ef6667407a63e485429dae0c050d93355b426a9a10236c16be network default NAD default] DEL finished CNI request [openshift-image-registry/image-registry-84884f658b-pb2kz 68283857f83395ef6667407a63e485429dae0c050d93355b426a9a10236c16be network default NAD default], result "{}", err I0330 15:00:40.444122 3819 obj_retry.go:464] Detected object openshift-image-registry/image-registry-84884f658b-pb2kz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:00:40.444243 3819 pods.go:176] Deleting pod: openshift-image-registry/image-registry-84884f658b-pb2kz I0330 15:00:40.445731 3819 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-84884f658b-pb2kz, ips: 10.129.0.6 I0330 15:00:40.445768 3819 obj_retry.go:464] Detected object openshift-image-registry/image-registry-84884f658b-pb2kz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:00:40.452699 3819 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-68df98f979-xpzcb] creating logical port openshift-image-registry_image-registry-68df98f979-xpzcb for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:40.452840 3819 kube.go:256] Updating pod openshift-image-registry/image-registry-68df98f979-xpzcb I0330 15:00:40.469381 3819 pod.go:62] [openshift-image-registry/image-registry-68df98f979-xpzcb] pod update took 16.573392ms I0330 15:00:40.469410 3819 base_network_controller_pods.go:937] [default/openshift-image-registry/image-registry-68df98f979-xpzcb] addLogicalPort annotation time took 16.607617ms I0330 15:00:40.470737 3819 pods.go:271] [openshift-image-registry/image-registry-68df98f979-xpzcb] addLogicalPort took 18.055157ms, libovsdb time 890.886µs I0330 15:00:40.816297 3819 cni.go:403] [openshift-image-registry/image-registry-68df98f979-xpzcb 0964575aabaca7cf0fd26aac902ac1b9cf6523780b18c8a79fafd193763f35a3 network default NAD default] ADD starting CNI request [openshift-image-registry/image-registry-68df98f979-xpzcb 0964575aabaca7cf0fd26aac902ac1b9cf6523780b18c8a79fafd193763f35a3 network default NAD default] I0330 15:00:40.825877 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-68df98f979-xpzcb, hostIfaceName: 0964575aabaca7c, network: default, NAD default, SandboxID: "0964575aabaca7cf0fd26aac902ac1b9cf6523780b18c8a79fafd193763f35a3", PCI device ID: , UID: "81fba4ab-b15c-43ed-8282-07c8810c3996", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0330 15:00:40.875379 3819 cni.go:424] [openshift-image-registry/image-registry-68df98f979-xpzcb 0964575aabaca7cf0fd26aac902ac1b9cf6523780b18c8a79fafd193763f35a3 network default NAD default] ADD finished CNI request [openshift-image-registry/image-registry-68df98f979-xpzcb 0964575aabaca7cf0fd26aac902ac1b9cf6523780b18c8a79fafd193763f35a3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"5a:fe:0b:4d:44:d0\",\"name\":\"0964575aabaca7c\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/367a9568-3c55-4c86-8de9-baa5ef46d435\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:00:41.380105 3819 obj_retry.go:458] Detected object openshift-image-registry/image-registry-84884f658b-pb2kz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:00:41.380133 3819 obj_retry.go:458] Detected object openshift-image-registry/image-registry-84884f658b-pb2kz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:00:41.385883 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-84884f658b-pb2kz I0330 15:00:41.385917 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-84884f658b-pb2kz I0330 15:00:41.856888 3819 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d] creating logical port openshift-deployment-validation-operator_ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:41.857014 3819 kube.go:256] Updating pod openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d I0330 15:00:41.878004 3819 pod.go:62] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d] pod update took 21.024552ms I0330 15:00:41.878025 3819 base_network_controller_pods.go:937] [default/openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d] addLogicalPort annotation time took 21.054036ms I0330 15:00:41.879369 3819 pods.go:271] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d] addLogicalPort took 22.49665ms, libovsdb time 937.712µs I0330 15:00:42.226025 3819 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d ce4a2680e0bef05cdaabf3df2aace274ca15f64e314eefa1372095f88e6a3aae network default NAD default] ADD starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d ce4a2680e0bef05cdaabf3df2aace274ca15f64e314eefa1372095f88e6a3aae network default NAD default] I0330 15:00:42.236085 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d, hostIfaceName: ce4a2680e0bef05, network: default, NAD default, SandboxID: "ce4a2680e0bef05cdaabf3df2aace274ca15f64e314eefa1372095f88e6a3aae", PCI device ID: , UID: "7dfdfe5b-561d-4012-a861-74f8da8e2040", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0330 15:00:42.281432 3819 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d ce4a2680e0bef05cdaabf3df2aace274ca15f64e314eefa1372095f88e6a3aae network default NAD default] ADD finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d ce4a2680e0bef05cdaabf3df2aace274ca15f64e314eefa1372095f88e6a3aae network default NAD default], result "{\"interfaces\":[{\"mac\":\"36:c6:f1:40:59:9d\",\"name\":\"ce4a2680e0bef05\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8b5473da-18a1-4446-be60-113ef256fafd\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:00:42.561194 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0330 15:00:42.561220 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0330 15:00:42.603497 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0330 15:00:42.603556 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0330 15:00:45.370742 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:45.370766 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:45.370773 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0330 15:00:45.370781 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0330 15:00:47.093619 3819 obj_retry.go:464] Detected object openshift-console/console-6c6664555d-jjpqj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:00:47.094239 3819 obj_retry.go:464] Detected object openshift-console/console-6c6664555d-jjpqj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:00:47.932356 3819 obj_retry.go:458] Detected object openshift-console/console-6c6664555d-jjpqj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:00:47.932380 3819 obj_retry.go:458] Detected object openshift-console/console-6c6664555d-jjpqj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:00:47.935262 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6c6664555d-jjpqj I0330 15:00:47.935298 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6c6664555d-jjpqj W0330 15:00:52.939214 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics I0330 15:00:56.834579 3819 cni.go:403] [openshift-console/console-6c6664555d-w9m8v 8ad73945d6ca1ffdb97fc04fdbaa528abdd64ad7d4fe4e391b01379c9b11c9ab network default NAD default] DEL starting CNI request [openshift-console/console-6c6664555d-w9m8v 8ad73945d6ca1ffdb97fc04fdbaa528abdd64ad7d4fe4e391b01379c9b11c9ab network default NAD default] I0330 15:00:56.914615 3819 cni.go:424] [openshift-console/console-6c6664555d-w9m8v 8ad73945d6ca1ffdb97fc04fdbaa528abdd64ad7d4fe4e391b01379c9b11c9ab network default NAD default] DEL finished CNI request [openshift-console/console-6c6664555d-w9m8v 8ad73945d6ca1ffdb97fc04fdbaa528abdd64ad7d4fe4e391b01379c9b11c9ab network default NAD default], result "{}", err I0330 15:00:57.007327 3819 obj_retry.go:464] Detected object openshift-console/console-6c6664555d-w9m8v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:00:57.007365 3819 pods.go:176] Deleting pod: openshift-console/console-6c6664555d-w9m8v I0330 15:00:57.008601 3819 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6c6664555d-w9m8v, ips: 10.129.0.10 I0330 15:00:57.008635 3819 obj_retry.go:464] Detected object openshift-console/console-6c6664555d-w9m8v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:00:57.016715 3819 base_network_controller_pods.go:487] [default/openshift-console/console-5cbf99c968-jh2x2] creating logical port openshift-console_console-5cbf99c968-jh2x2 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:00:57.016839 3819 kube.go:256] Updating pod openshift-console/console-5cbf99c968-jh2x2 I0330 15:00:57.031651 3819 pod.go:62] [openshift-console/console-5cbf99c968-jh2x2] pod update took 14.851448ms I0330 15:00:57.031681 3819 base_network_controller_pods.go:937] [default/openshift-console/console-5cbf99c968-jh2x2] addLogicalPort annotation time took 14.884642ms I0330 15:00:57.033168 3819 pods.go:271] [openshift-console/console-5cbf99c968-jh2x2] addLogicalPort took 16.469631ms, libovsdb time 1.140207ms I0330 15:00:57.419562 3819 cni.go:403] [openshift-console/console-5cbf99c968-jh2x2 b28ff429dfe3c6f4abfbd10e6a9e5a4e9cff291ee1f3b14c8d0ebad02de31570 network default NAD default] ADD starting CNI request [openshift-console/console-5cbf99c968-jh2x2 b28ff429dfe3c6f4abfbd10e6a9e5a4e9cff291ee1f3b14c8d0ebad02de31570 network default NAD default] I0330 15:00:57.429939 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-5cbf99c968-jh2x2, hostIfaceName: b28ff429dfe3c6f, network: default, NAD default, SandboxID: "b28ff429dfe3c6f4abfbd10e6a9e5a4e9cff291ee1f3b14c8d0ebad02de31570", PCI device ID: , UID: "aafa4d4b-a7f9-42e1-9824-7d67c6b1143d", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0330 15:00:57.439423 3819 obj_retry.go:458] Detected object openshift-console/console-6c6664555d-w9m8v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:00:57.439464 3819 obj_retry.go:458] Detected object openshift-console/console-6c6664555d-w9m8v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:00:57.444883 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6c6664555d-w9m8v I0330 15:00:57.444915 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6c6664555d-w9m8v I0330 15:00:57.484301 3819 cni.go:424] [openshift-console/console-5cbf99c968-jh2x2 b28ff429dfe3c6f4abfbd10e6a9e5a4e9cff291ee1f3b14c8d0ebad02de31570 network default NAD default] ADD finished CNI request [openshift-console/console-5cbf99c968-jh2x2 b28ff429dfe3c6f4abfbd10e6a9e5a4e9cff291ee1f3b14c8d0ebad02de31570 network default NAD default], result "{\"interfaces\":[{\"mac\":\"c6:5a:00:77:76:ac\",\"name\":\"b28ff429dfe3c6f\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/80bf94cb-afe5-4307-bae9-c9d645b77283\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:00:59.348948 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0330 15:01:02.374860 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0330 15:01:02.511509 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:01:02.511554 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:01:02.511563 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:01:02.511580 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:01:02.511587 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:01:02.511594 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:01:02.528730 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:01:02.528749 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:01:02.528755 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0330 15:01:02.981314 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0330 15:01:03.475912 3819 cni.go:403] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d ce4a2680e0bef05cdaabf3df2aace274ca15f64e314eefa1372095f88e6a3aae network default NAD default] DEL starting CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d ce4a2680e0bef05cdaabf3df2aace274ca15f64e314eefa1372095f88e6a3aae network default NAD default] I0330 15:01:03.554040 3819 cni.go:424] [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d ce4a2680e0bef05cdaabf3df2aace274ca15f64e314eefa1372095f88e6a3aae network default NAD default] DEL finished CNI request [openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d ce4a2680e0bef05cdaabf3df2aace274ca15f64e314eefa1372095f88e6a3aae network default NAD default], result "{}", err I0330 15:01:03.630866 3819 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:03.630909 3819 pods.go:176] Deleting pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d I0330 15:01:03.632349 3819 pods.go:236] Attempting to release IPs for pod: openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d, ips: 10.129.0.21 I0330 15:01:03.632384 3819 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:03.632408 3819 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:01:03.632418 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0330 15:01:03.632654 3819 obj_retry.go:464] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:01:03.632672 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0330 15:01:04.657094 3819 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:04.657121 3819 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:04.657136 3819 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:04.657151 3819 obj_retry.go:458] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf56539079w4d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:06.938376 3819 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0330 15:01:06.947877 3819 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0330 15:01:07.497496 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0330 15:01:07.917025 3819 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7967d4f498-p5v84 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:07.917663 3819 obj_retry.go:464] Detected object openshift-image-registry/image-registry-7967d4f498-p5v84 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:08.309654 3819 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7967d4f498-p5v84 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:08.309680 3819 obj_retry.go:458] Detected object openshift-image-registry/image-registry-7967d4f498-p5v84 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:08.314810 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7967d4f498-p5v84 I0330 15:01:08.314847 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7967d4f498-p5v84 W0330 15:01:13.041200 3819 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0330 15:01:23.084655 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0330 15:01:23.274868 3819 obj_retry.go:464] Detected object openshift-console/console-7db448cdd6-h6gtj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:23.275438 3819 obj_retry.go:464] Detected object openshift-console/console-7db448cdd6-h6gtj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:23.858136 3819 obj_retry.go:458] Detected object openshift-console/console-7db448cdd6-h6gtj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:23.858163 3819 obj_retry.go:458] Detected object openshift-console/console-7db448cdd6-h6gtj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:23.863886 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7db448cdd6-h6gtj I0330 15:01:23.863916 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7db448cdd6-h6gtj I0330 15:01:24.343325 3819 obj_retry.go:464] Detected object openshift-ingress/router-default-79cb556bcf-kl4ns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:24.343941 3819 obj_retry.go:464] Detected object openshift-ingress/router-default-79cb556bcf-kl4ns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:25.090144 3819 obj_retry.go:458] Detected object openshift-ingress/router-default-79cb556bcf-kl4ns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:25.090177 3819 obj_retry.go:458] Detected object openshift-ingress/router-default-79cb556bcf-kl4ns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:25.092885 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-79cb556bcf-kl4ns I0330 15:01:25.092899 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-79cb556bcf-kl4ns W0330 15:01:37.942185 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0330 15:01:37.942212 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0330 15:01:37.942229 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0330 15:01:37.942238 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0330 15:01:38.650023 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0330 15:01:38.650138 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0330 15:01:49.667320 3819 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd 769bed655bb450a4bbf4e0aa8e89ed0cad07e4210f5038460040c2eee1c89d02 network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd 769bed655bb450a4bbf4e0aa8e89ed0cad07e4210f5038460040c2eee1c89d02 network default NAD default] I0330 15:01:49.750089 3819 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd 769bed655bb450a4bbf4e0aa8e89ed0cad07e4210f5038460040c2eee1c89d02 network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd 769bed655bb450a4bbf4e0aa8e89ed0cad07e4210f5038460040c2eee1c89d02 network default NAD default], result "{}", err I0330 15:01:49.791947 3819 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:49.791984 3819 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd I0330 15:01:49.793278 3819 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd, ips: 10.129.0.13 I0330 15:01:49.793313 3819 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:01:50.675894 3819 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:01:50.675920 3819 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29581380-ssrkd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0330 15:02:10.216905 3819 base_network_controller_namespace.go:457] Unable to remove remote zone pod's openshift-console/console-86d7b5858c-4gp79 IP address from the namespace address-set, err: pod openshift-console/console-86d7b5858c-4gp79: no pod IPs found I0330 15:02:24.602966 3819 obj_retry.go:464] Detected object openshift-console/console-5cbf99c968-bw4b6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:02:24.603545 3819 obj_retry.go:464] Detected object openshift-console/console-5cbf99c968-bw4b6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:02:25.371296 3819 obj_retry.go:458] Detected object openshift-console/console-5cbf99c968-bw4b6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:02:25.371320 3819 obj_retry.go:458] Detected object openshift-console/console-5cbf99c968-bw4b6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:02:25.376231 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5cbf99c968-bw4b6 I0330 15:02:25.376254 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5cbf99c968-bw4b6 I0330 15:02:35.246331 3819 cni.go:403] [openshift-console/console-5cbf99c968-jh2x2 b28ff429dfe3c6f4abfbd10e6a9e5a4e9cff291ee1f3b14c8d0ebad02de31570 network default NAD default] DEL starting CNI request [openshift-console/console-5cbf99c968-jh2x2 b28ff429dfe3c6f4abfbd10e6a9e5a4e9cff291ee1f3b14c8d0ebad02de31570 network default NAD default] I0330 15:02:35.322699 3819 cni.go:424] [openshift-console/console-5cbf99c968-jh2x2 b28ff429dfe3c6f4abfbd10e6a9e5a4e9cff291ee1f3b14c8d0ebad02de31570 network default NAD default] DEL finished CNI request [openshift-console/console-5cbf99c968-jh2x2 b28ff429dfe3c6f4abfbd10e6a9e5a4e9cff291ee1f3b14c8d0ebad02de31570 network default NAD default], result "{}", err I0330 15:02:35.370937 3819 obj_retry.go:464] Detected object openshift-console/console-5cbf99c968-jh2x2 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:02:35.370968 3819 pods.go:176] Deleting pod: openshift-console/console-5cbf99c968-jh2x2 I0330 15:02:35.372112 3819 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5cbf99c968-jh2x2, ips: 10.129.0.22 I0330 15:02:35.372141 3819 obj_retry.go:464] Detected object openshift-console/console-5cbf99c968-jh2x2 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:02:35.380857 3819 base_network_controller_pods.go:487] [default/openshift-console/console-55b9676c48-6ttqz] creating logical port openshift-console_console-55b9676c48-6ttqz for pod on switch ip-10-0-1-27.ec2.internal I0330 15:02:35.381026 3819 kube.go:256] Updating pod openshift-console/console-55b9676c48-6ttqz I0330 15:02:35.395389 3819 pod.go:62] [openshift-console/console-55b9676c48-6ttqz] pod update took 14.405752ms I0330 15:02:35.395420 3819 base_network_controller_pods.go:937] [default/openshift-console/console-55b9676c48-6ttqz] addLogicalPort annotation time took 14.441152ms I0330 15:02:35.396618 3819 pods.go:271] [openshift-console/console-55b9676c48-6ttqz] addLogicalPort took 15.781803ms, libovsdb time 859.252µs I0330 15:02:35.740159 3819 cni.go:403] [openshift-console/console-55b9676c48-6ttqz ea1c9c99785a3b4e1882dc1fc7dbc5e86d43c18cb6d7bac23b5f382f6fffd0b1 network default NAD default] ADD starting CNI request [openshift-console/console-55b9676c48-6ttqz ea1c9c99785a3b4e1882dc1fc7dbc5e86d43c18cb6d7bac23b5f382f6fffd0b1 network default NAD default] I0330 15:02:35.748569 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-console, podName: console-55b9676c48-6ttqz, hostIfaceName: ea1c9c99785a3b4, network: default, NAD default, SandboxID: "ea1c9c99785a3b4e1882dc1fc7dbc5e86d43c18cb6d7bac23b5f382f6fffd0b1", PCI device ID: , UID: "dd42aea2-5c55-4b23-9630-b9c0ea902015", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0330 15:02:35.793789 3819 cni.go:424] [openshift-console/console-55b9676c48-6ttqz ea1c9c99785a3b4e1882dc1fc7dbc5e86d43c18cb6d7bac23b5f382f6fffd0b1 network default NAD default] ADD finished CNI request [openshift-console/console-55b9676c48-6ttqz ea1c9c99785a3b4e1882dc1fc7dbc5e86d43c18cb6d7bac23b5f382f6fffd0b1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:5f:43:12:d1:6a\",\"name\":\"ea1c9c99785a3b4\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5cdad0f8-492c-4dca-8724-376d236f2d7f\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:02:35.843604 3819 obj_retry.go:458] Detected object openshift-console/console-5cbf99c968-jh2x2 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:02:35.843629 3819 obj_retry.go:458] Detected object openshift-console/console-5cbf99c968-jh2x2 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:02:35.849132 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5cbf99c968-jh2x2 I0330 15:02:35.849198 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5cbf99c968-jh2x2 W0330 15:02:45.906303 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0330 15:03:00.732380 3819 obj_retry.go:464] Detected object openshift-console/console-86d7b5858c-dx7pf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:03:00.732999 3819 obj_retry.go:464] Detected object openshift-console/console-86d7b5858c-dx7pf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:03:01.233946 3819 obj_retry.go:458] Detected object openshift-console/console-86d7b5858c-dx7pf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:03:01.233970 3819 obj_retry.go:458] Detected object openshift-console/console-86d7b5858c-dx7pf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:03:01.240839 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-86d7b5858c-dx7pf I0330 15:03:01.240864 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-86d7b5858c-dx7pf I0330 15:03:58.487172 3819 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-65.ec2.internal I0330 15:03:58.487213 3819 egressfirewall.go:628] Syncing node "ip-10-0-0-65.ec2.internal" for egress firewall I0330 15:03:58.487228 3819 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-65.ec2.internal: 64.024µs I0330 15:04:22.598780 3819 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0330 15:05:27.164067 3819 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0330 15:05:50.170016 3819 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=7 I0330 15:05:51.139193 3819 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0330 15:05:57.151574 3819 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0330 15:06:21.133457 3819 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0330 15:06:34.344749 3819 namespace.go:98] [tsf] adding namespace I0330 15:06:34.345650 3819 namespace.go:102] [tsf] adding namespace took 874.456µs I0330 15:06:34.360132 3819 namespace.go:142] [tsf] updating namespace I0330 15:06:34.371215 3819 namespace.go:142] [tsf] updating namespace I0330 15:06:38.158342 3819 namespace.go:98] [cert-manager-operator] adding namespace I0330 15:06:38.159209 3819 namespace.go:102] [cert-manager-operator] adding namespace took 840.703µs I0330 15:06:38.164958 3819 namespace.go:98] [konflux-operator] adding namespace I0330 15:06:38.165014 3819 namespace.go:98] [tsf-tas] adding namespace I0330 15:06:38.165071 3819 namespace.go:98] [tsf-keycloak] adding namespace I0330 15:06:38.165203 3819 namespace.go:98] [tsf-tpa] adding namespace I0330 15:06:38.165275 3819 namespace.go:98] [rhtpa-operator] adding namespace I0330 15:06:38.165357 3819 namespace.go:98] [konflux-ui] adding namespace I0330 15:06:38.165833 3819 namespace.go:102] [konflux-operator] adding namespace took 855.255µs I0330 15:06:38.166452 3819 namespace.go:102] [tsf-tas] adding namespace took 1.419867ms I0330 15:06:38.167137 3819 namespace.go:102] [tsf-keycloak] adding namespace took 2.051868ms I0330 15:06:38.167693 3819 namespace.go:102] [tsf-tpa] adding namespace took 2.473404ms I0330 15:06:38.167715 3819 namespace.go:98] [rhbk-operator] adding namespace I0330 15:06:38.168346 3819 namespace.go:102] [rhtpa-operator] adding namespace took 3.059589ms I0330 15:06:38.169042 3819 namespace.go:102] [konflux-ui] adding namespace took 3.668619ms I0330 15:06:38.169648 3819 namespace.go:102] [rhbk-operator] adding namespace took 1.925931ms I0330 15:06:38.180577 3819 namespace.go:142] [cert-manager-operator] updating namespace I0330 15:06:38.197853 3819 namespace.go:142] [cert-manager-operator] updating namespace I0330 15:06:38.207146 3819 namespace.go:142] [konflux-operator] updating namespace I0330 15:06:38.210670 3819 namespace.go:142] [tsf-tas] updating namespace I0330 15:06:38.210732 3819 namespace.go:142] [tsf-keycloak] updating namespace I0330 15:06:38.212445 3819 namespace.go:142] [tsf-tpa] updating namespace I0330 15:06:38.218108 3819 namespace.go:142] [rhtpa-operator] updating namespace I0330 15:06:38.224515 3819 namespace.go:142] [rhbk-operator] updating namespace I0330 15:06:38.224653 3819 namespace.go:142] [konflux-ui] updating namespace I0330 15:06:38.229121 3819 namespace.go:142] [cert-manager-operator] updating namespace I0330 15:06:38.269821 3819 namespace.go:142] [tsf-keycloak] updating namespace I0330 15:06:38.285710 3819 namespace.go:142] [tsf-keycloak] updating namespace I0330 15:06:38.288096 3819 namespace.go:142] [tsf-tpa] updating namespace I0330 15:06:38.302682 3819 namespace.go:142] [tsf-tpa] updating namespace I0330 15:06:38.311006 3819 namespace.go:142] [rhtpa-operator] updating namespace I0330 15:06:38.322365 3819 namespace.go:142] [rhtpa-operator] updating namespace I0330 15:06:38.330262 3819 namespace.go:142] [konflux-ui] updating namespace I0330 15:06:38.349379 3819 namespace.go:142] [konflux-ui] updating namespace I0330 15:06:38.349473 3819 namespace.go:142] [rhbk-operator] updating namespace I0330 15:06:38.360567 3819 namespace.go:142] [rhbk-operator] updating namespace I0330 15:06:38.371131 3819 namespace.go:142] [konflux-operator] updating namespace I0330 15:06:38.386839 3819 namespace.go:142] [konflux-operator] updating namespace I0330 15:06:38.573174 3819 namespace.go:142] [tsf-tas] updating namespace I0330 15:06:38.582527 3819 namespace.go:142] [tsf-tas] updating namespace I0330 15:06:39.317965 3819 namespace.go:142] [tsf-tpa] updating namespace I0330 15:06:39.323146 3819 namespace.go:142] [tsf-keycloak] updating namespace I0330 15:06:40.980918 3819 base_network_controller_pods.go:487] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh] creating logical port openshift-marketplace_ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh for pod on switch ip-10-0-1-27.ec2.internal I0330 15:06:40.981080 3819 kube.go:256] Updating pod openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh I0330 15:06:40.981838 3819 base_network_controller_pods.go:487] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl] creating logical port openshift-marketplace_925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl for pod on switch ip-10-0-1-27.ec2.internal I0330 15:06:40.981938 3819 kube.go:256] Updating pod openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl I0330 15:06:40.998945 3819 pod.go:62] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl] pod update took 17.03578ms I0330 15:06:40.998971 3819 base_network_controller_pods.go:937] [default/openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl] addLogicalPort annotation time took 17.065267ms I0330 15:06:40.998974 3819 pod.go:62] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh] pod update took 17.926059ms I0330 15:06:40.998992 3819 base_network_controller_pods.go:937] [default/openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh] addLogicalPort annotation time took 17.952122ms I0330 15:06:41.000188 3819 pods.go:271] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl] addLogicalPort took 18.359284ms, libovsdb time 878.946µs I0330 15:06:41.000748 3819 pods.go:271] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh] addLogicalPort took 19.853985ms, libovsdb time 1.251091ms I0330 15:06:41.128672 3819 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0330 15:06:41.343257 3819 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh acf46e2cc7deab5dcb4e11d9bf13deb47bd0f1247735adac20353c17f99d947f network default NAD default] ADD starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh acf46e2cc7deab5dcb4e11d9bf13deb47bd0f1247735adac20353c17f99d947f network default NAD default] I0330 15:06:41.347687 3819 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl be8630f6119b48cebb80efd415c9b79b1c6fb6dbf125e8e158cd32e5ec7c09e9 network default NAD default] ADD starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl be8630f6119b48cebb80efd415c9b79b1c6fb6dbf125e8e158cd32e5ec7c09e9 network default NAD default] I0330 15:06:41.353143 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh, hostIfaceName: acf46e2cc7deab5, network: default, NAD default, SandboxID: "acf46e2cc7deab5dcb4e11d9bf13deb47bd0f1247735adac20353c17f99d947f", PCI device ID: , UID: "130460a6-e015-441c-84e8-a5b76b013f4b", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0330 15:06:41.356810 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl, hostIfaceName: be8630f6119b48c, network: default, NAD default, SandboxID: "be8630f6119b48cebb80efd415c9b79b1c6fb6dbf125e8e158cd32e5ec7c09e9", PCI device ID: , UID: "15d605b8-eca7-4b46-a83b-55060eb6578b", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0330 15:06:41.398818 3819 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh acf46e2cc7deab5dcb4e11d9bf13deb47bd0f1247735adac20353c17f99d947f network default NAD default] ADD finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh acf46e2cc7deab5dcb4e11d9bf13deb47bd0f1247735adac20353c17f99d947f network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:9f:46:31:e2:e2\",\"name\":\"acf46e2cc7deab5\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bc4a8ab8-17cc-48c8-87c2-86982dd8235b\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:06:41.423162 3819 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl be8630f6119b48cebb80efd415c9b79b1c6fb6dbf125e8e158cd32e5ec7c09e9 network default NAD default] ADD finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl be8630f6119b48cebb80efd415c9b79b1c6fb6dbf125e8e158cd32e5ec7c09e9 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:76:44:6a:41:ec\",\"name\":\"be8630f6119b48c\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b046b901-9168-4f80-b08b-9d59f5484ac0\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:06:44.066247 3819 base_network_controller_pods.go:487] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf] creating logical port openshift-marketplace_3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf for pod on switch ip-10-0-1-27.ec2.internal I0330 15:06:44.066403 3819 kube.go:256] Updating pod openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf I0330 15:06:44.078816 3819 pod.go:62] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf] pod update took 12.448983ms I0330 15:06:44.078845 3819 base_network_controller_pods.go:937] [default/openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf] addLogicalPort annotation time took 12.483916ms I0330 15:06:44.080282 3819 pods.go:271] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf] addLogicalPort took 14.058312ms, libovsdb time 1.069994ms I0330 15:06:44.485780 3819 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf 9afbf77b14524e1d20ba94f2ee2e9b9e41a601515147f563b67c1b0b181f1357 network default NAD default] ADD starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf 9afbf77b14524e1d20ba94f2ee2e9b9e41a601515147f563b67c1b0b181f1357 network default NAD default] I0330 15:06:44.495264 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf, hostIfaceName: 9afbf77b14524e1, network: default, NAD default, SandboxID: "9afbf77b14524e1d20ba94f2ee2e9b9e41a601515147f563b67c1b0b181f1357", PCI device ID: , UID: "32535c9a-7afe-4c23-afec-74bf39078373", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0330 15:06:44.541978 3819 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf 9afbf77b14524e1d20ba94f2ee2e9b9e41a601515147f563b67c1b0b181f1357 network default NAD default] ADD finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf 9afbf77b14524e1d20ba94f2ee2e9b9e41a601515147f563b67c1b0b181f1357 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:39:21:f2:a2:3e\",\"name\":\"9afbf77b14524e1\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ac0af17b-c245-4a20-a2b0-488d59702581\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:06:45.459868 3819 base_network_controller_pods.go:487] [default/openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc] creating logical port openshift-marketplace_867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc for pod on switch ip-10-0-1-27.ec2.internal I0330 15:06:45.459998 3819 kube.go:256] Updating pod openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc I0330 15:06:45.471896 3819 pod.go:62] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc] pod update took 11.92635ms I0330 15:06:45.471927 3819 base_network_controller_pods.go:937] [default/openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc] addLogicalPort annotation time took 11.963386ms I0330 15:06:45.473146 3819 pods.go:271] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc] addLogicalPort took 13.296126ms, libovsdb time 852.176µs I0330 15:06:45.820560 3819 cni.go:403] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc 53e798589e51e9fb32f61fce241a7a0c97b171b3906658b38ebcfe934ce22844 network default NAD default] ADD starting CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc 53e798589e51e9fb32f61fce241a7a0c97b171b3906658b38ebcfe934ce22844 network default NAD default] I0330 15:06:45.830222 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc, hostIfaceName: 53e798589e51e9f, network: default, NAD default, SandboxID: "53e798589e51e9fb32f61fce241a7a0c97b171b3906658b38ebcfe934ce22844", PCI device ID: , UID: "96611789-1e35-40f5-aaf0-867a0ec01e97", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0330 15:06:45.878982 3819 cni.go:424] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc 53e798589e51e9fb32f61fce241a7a0c97b171b3906658b38ebcfe934ce22844 network default NAD default] ADD finished CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc 53e798589e51e9fb32f61fce241a7a0c97b171b3906658b38ebcfe934ce22844 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:a1:e9:c7:eb:28\",\"name\":\"53e798589e51e9f\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e9008ea9-a527-4419-9b2b-276d3be85db7\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:06:45.896451 3819 cni.go:403] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh acf46e2cc7deab5dcb4e11d9bf13deb47bd0f1247735adac20353c17f99d947f network default NAD default] DEL starting CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh acf46e2cc7deab5dcb4e11d9bf13deb47bd0f1247735adac20353c17f99d947f network default NAD default] I0330 15:06:45.972292 3819 cni.go:424] [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh acf46e2cc7deab5dcb4e11d9bf13deb47bd0f1247735adac20353c17f99d947f network default NAD default] DEL finished CNI request [openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh acf46e2cc7deab5dcb4e11d9bf13deb47bd0f1247735adac20353c17f99d947f network default NAD default], result "{}", err I0330 15:06:46.024229 3819 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:46.024294 3819 pods.go:176] Deleting pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh I0330 15:06:46.026054 3819 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh, ips: 10.129.0.24 I0330 15:06:46.026093 3819 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:46.026121 3819 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:46.026134 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0330 15:06:46.026449 3819 obj_retry.go:464] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:46.026468 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0330 15:06:46.905931 3819 cni.go:403] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl be8630f6119b48cebb80efd415c9b79b1c6fb6dbf125e8e158cd32e5ec7c09e9 network default NAD default] DEL starting CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl be8630f6119b48cebb80efd415c9b79b1c6fb6dbf125e8e158cd32e5ec7c09e9 network default NAD default] I0330 15:06:46.993391 3819 cni.go:424] [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl be8630f6119b48cebb80efd415c9b79b1c6fb6dbf125e8e158cd32e5ec7c09e9 network default NAD default] DEL finished CNI request [openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl be8630f6119b48cebb80efd415c9b79b1c6fb6dbf125e8e158cd32e5ec7c09e9 network default NAD default], result "{}", err I0330 15:06:47.054290 3819 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:47.054339 3819 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:47.054354 3819 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:47.054368 3819 obj_retry.go:458] Detected object openshift-marketplace/ba8005e542d2434eb921fc7d1b9a173d62cb93ec352f86c52b78a87ba86z7sh of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:47.091873 3819 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:47.091916 3819 pods.go:176] Deleting pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl I0330 15:06:47.093592 3819 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl, ips: 10.129.0.25 I0330 15:06:47.093619 3819 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:47.093644 3819 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:47.093652 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0330 15:06:47.093846 3819 obj_retry.go:464] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:47.093866 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0330 15:06:47.137902 3819 base_network_controller_pods.go:487] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg] creating logical port openshift-marketplace_30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg for pod on switch ip-10-0-1-27.ec2.internal I0330 15:06:47.138168 3819 kube.go:256] Updating pod openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg I0330 15:06:47.150757 3819 pod.go:62] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg] pod update took 12.63771ms I0330 15:06:47.150783 3819 base_network_controller_pods.go:937] [default/openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg] addLogicalPort annotation time took 12.669613ms I0330 15:06:47.152260 3819 pods.go:271] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg] addLogicalPort took 14.39455ms, libovsdb time 1.134355ms I0330 15:06:47.232817 3819 base_network_controller_pods.go:487] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5] creating logical port openshift-marketplace_18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:06:47.232935 3819 kube.go:256] Updating pod openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 I0330 15:06:47.245739 3819 pod.go:62] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5] pod update took 12.828276ms I0330 15:06:47.245852 3819 base_network_controller_pods.go:937] [default/openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5] addLogicalPort annotation time took 12.944424ms I0330 15:06:47.247151 3819 pods.go:271] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5] addLogicalPort took 14.34536ms, libovsdb time 941.28µs I0330 15:06:47.542219 3819 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg 3f2bf330217a4ec39f4aef60c6c4d6a0c2549ead0cb221930a4546abf37496fa network default NAD default] ADD starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg 3f2bf330217a4ec39f4aef60c6c4d6a0c2549ead0cb221930a4546abf37496fa network default NAD default] I0330 15:06:47.552433 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg, hostIfaceName: 3f2bf330217a4ec, network: default, NAD default, SandboxID: "3f2bf330217a4ec39f4aef60c6c4d6a0c2549ead0cb221930a4546abf37496fa", PCI device ID: , UID: "80cd69ea-4a3a-4315-ab08-8e66263abf53", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0330 15:06:47.599306 3819 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 e3f1e73e691a7a6e6866558e300ec15bea5bf1abee44e2fafb9a3ffd2cf2cffd network default NAD default] ADD starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 e3f1e73e691a7a6e6866558e300ec15bea5bf1abee44e2fafb9a3ffd2cf2cffd network default NAD default] I0330 15:06:47.600630 3819 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg 3f2bf330217a4ec39f4aef60c6c4d6a0c2549ead0cb221930a4546abf37496fa network default NAD default] ADD finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg 3f2bf330217a4ec39f4aef60c6c4d6a0c2549ead0cb221930a4546abf37496fa network default NAD default], result "{\"interfaces\":[{\"mac\":\"aa:af:c0:47:d1:2a\",\"name\":\"3f2bf330217a4ec\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/58d970b8-6ecc-4a2e-b906-934e81604eed\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:06:47.607767 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-marketplace, podName: 18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5, hostIfaceName: e3f1e73e691a7a6, network: default, NAD default, SandboxID: "e3f1e73e691a7a6e6866558e300ec15bea5bf1abee44e2fafb9a3ffd2cf2cffd", PCI device ID: , UID: "1a1b8a37-cd4c-4a4b-8fd1-3c739f61e1c2", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0330 15:06:47.657394 3819 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 e3f1e73e691a7a6e6866558e300ec15bea5bf1abee44e2fafb9a3ffd2cf2cffd network default NAD default] ADD finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 e3f1e73e691a7a6e6866558e300ec15bea5bf1abee44e2fafb9a3ffd2cf2cffd network default NAD default], result "{\"interfaces\":[{\"mac\":\"a6:8b:32:2b:d3:9b\",\"name\":\"e3f1e73e691a7a6\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1cd275eb-6a31-47e8-957c-fdb3b4042bda\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:06:48.122738 3819 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:48.122785 3819 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:48.122800 3819 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:48.122811 3819 obj_retry.go:458] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5q74dl of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:48.923438 3819 cni.go:403] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf 9afbf77b14524e1d20ba94f2ee2e9b9e41a601515147f563b67c1b0b181f1357 network default NAD default] DEL starting CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf 9afbf77b14524e1d20ba94f2ee2e9b9e41a601515147f563b67c1b0b181f1357 network default NAD default] I0330 15:06:49.000892 3819 cni.go:424] [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf 9afbf77b14524e1d20ba94f2ee2e9b9e41a601515147f563b67c1b0b181f1357 network default NAD default] DEL finished CNI request [openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf 9afbf77b14524e1d20ba94f2ee2e9b9e41a601515147f563b67c1b0b181f1357 network default NAD default], result "{}", err I0330 15:06:49.049407 3819 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:49.049443 3819 pods.go:176] Deleting pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf I0330 15:06:49.051030 3819 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf, ips: 10.129.0.26 I0330 15:06:49.051062 3819 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:49.051091 3819 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:49.051102 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0330 15:06:49.051317 3819 obj_retry.go:464] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:49.051332 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0330 15:06:49.936419 3819 cni.go:403] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc 53e798589e51e9fb32f61fce241a7a0c97b171b3906658b38ebcfe934ce22844 network default NAD default] DEL starting CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc 53e798589e51e9fb32f61fce241a7a0c97b171b3906658b38ebcfe934ce22844 network default NAD default] I0330 15:06:50.022698 3819 cni.go:424] [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc 53e798589e51e9fb32f61fce241a7a0c97b171b3906658b38ebcfe934ce22844 network default NAD default] DEL finished CNI request [openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc 53e798589e51e9fb32f61fce241a7a0c97b171b3906658b38ebcfe934ce22844 network default NAD default], result "{}", err I0330 15:06:50.078253 3819 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:50.078286 3819 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:50.078308 3819 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:50.078319 3819 obj_retry.go:458] Detected object openshift-marketplace/3e5f5b636bd3cc990d4bd6297d76653ce1cee577740080d10b92cd17d16dwnf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:50.128279 3819 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0330 15:06:50.184878 3819 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:50.184913 3819 pods.go:176] Deleting pod: openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc I0330 15:06:50.186222 3819 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc, ips: 10.129.0.27 I0330 15:06:50.186247 3819 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:50.186263 3819 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:50.186272 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0330 15:06:50.186447 3819 obj_retry.go:464] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:50.186463 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0330 15:06:51.214511 3819 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:51.214583 3819 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:51.214600 3819 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:51.214619 3819 obj_retry.go:458] Detected object openshift-marketplace/867f63058f8a38d9051044b43e55794b351c4868f3d223f49663e800bf4k2vc of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:51.949319 3819 cni.go:403] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 e3f1e73e691a7a6e6866558e300ec15bea5bf1abee44e2fafb9a3ffd2cf2cffd network default NAD default] DEL starting CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 e3f1e73e691a7a6e6866558e300ec15bea5bf1abee44e2fafb9a3ffd2cf2cffd network default NAD default] I0330 15:06:51.949347 3819 cni.go:403] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg 3f2bf330217a4ec39f4aef60c6c4d6a0c2549ead0cb221930a4546abf37496fa network default NAD default] DEL starting CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg 3f2bf330217a4ec39f4aef60c6c4d6a0c2549ead0cb221930a4546abf37496fa network default NAD default] I0330 15:06:52.029573 3819 cni.go:424] [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg 3f2bf330217a4ec39f4aef60c6c4d6a0c2549ead0cb221930a4546abf37496fa network default NAD default] DEL finished CNI request [openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg 3f2bf330217a4ec39f4aef60c6c4d6a0c2549ead0cb221930a4546abf37496fa network default NAD default], result "{}", err I0330 15:06:52.053157 3819 cni.go:424] [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 e3f1e73e691a7a6e6866558e300ec15bea5bf1abee44e2fafb9a3ffd2cf2cffd network default NAD default] DEL finished CNI request [openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 e3f1e73e691a7a6e6866558e300ec15bea5bf1abee44e2fafb9a3ffd2cf2cffd network default NAD default], result "{}", err I0330 15:06:52.077067 3819 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:52.077108 3819 pods.go:176] Deleting pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg I0330 15:06:52.078558 3819 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg, ips: 10.129.0.28 I0330 15:06:52.078596 3819 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:52.078622 3819 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:52.078632 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0330 15:06:52.078835 3819 obj_retry.go:464] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:52.078853 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0330 15:06:52.101314 3819 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:52.101350 3819 pods.go:176] Deleting pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 I0330 15:06:52.102715 3819 pods.go:236] Attempting to release IPs for pod: openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5, ips: 10.129.0.29 I0330 15:06:52.102752 3819 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:06:52.102773 3819 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:52.102782 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0330 15:06:52.103021 3819 obj_retry.go:464] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0330 15:06:52.103039 3819 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0330 15:06:53.041312 3819 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=58 I0330 15:06:53.101548 3819 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:53.101599 3819 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:53.101615 3819 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:53.101629 3819 obj_retry.go:458] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaaf7vwwg of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:53.129962 3819 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:53.129989 3819 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:53.130007 3819 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:53.130021 3819 obj_retry.go:458] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7k8rl5 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:06:53.293620 3819 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-lmswh] creating logical port tsf-tpa_rhtpa-operator-controller-manager-746944f6d7-lmswh for pod on switch ip-10-0-1-27.ec2.internal I0330 15:06:53.293771 3819 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-lmswh I0330 15:06:53.310434 3819 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-lmswh] pod update took 16.68774ms I0330 15:06:53.310460 3819 base_network_controller_pods.go:937] [default/tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-lmswh] addLogicalPort annotation time took 16.718851ms I0330 15:06:53.311795 3819 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-lmswh] addLogicalPort took 18.19625ms, libovsdb time 990.685µs I0330 15:06:53.657552 3819 cni.go:403] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-lmswh 35b69054ac71c29acedd7a47e4b3a09ee69d0e2ec33bd58861f31baa3220e9be network default NAD default] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-lmswh 35b69054ac71c29acedd7a47e4b3a09ee69d0e2ec33bd58861f31baa3220e9be network default NAD default] I0330 15:06:53.666295 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-746944f6d7-lmswh, hostIfaceName: 35b69054ac71c29, network: default, NAD default, SandboxID: "35b69054ac71c29acedd7a47e4b3a09ee69d0e2ec33bd58861f31baa3220e9be", PCI device ID: , UID: "8ce5f91b-19d6-4cf3-895c-774e75c6e8ac", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0330 15:06:53.918666 3819 cni.go:424] [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-lmswh 35b69054ac71c29acedd7a47e4b3a09ee69d0e2ec33bd58861f31baa3220e9be network default NAD default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-746944f6d7-lmswh 35b69054ac71c29acedd7a47e4b3a09ee69d0e2ec33bd58861f31baa3220e9be network default NAD default], result "{\"interfaces\":[{\"mac\":\"2e:b1:a0:75:65:87\",\"name\":\"35b69054ac71c29\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ab08863e-2532-4bb0-9979-29d36318ff0d\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:06:56.976418 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0330 15:06:57.352796 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0330 15:06:57.369721 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0330 15:06:57.998472 3819 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-jplmx] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-58df4cf46c-jplmx for pod on switch ip-10-0-1-27.ec2.internal I0330 15:06:57.998661 3819 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-jplmx I0330 15:06:58.017989 3819 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-jplmx] pod update took 19.359377ms I0330 15:06:58.018018 3819 base_network_controller_pods.go:937] [default/cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-jplmx] addLogicalPort annotation time took 19.392702ms I0330 15:06:58.019338 3819 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-jplmx] addLogicalPort took 20.88272ms, libovsdb time 1.002021ms I0330 15:06:58.358510 3819 cni.go:403] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-jplmx b00dafb517eed625a2eed981937fc4b4a3c0fba9e86799d9ff53dc9541098223 network default NAD default] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-jplmx b00dafb517eed625a2eed981937fc4b4a3c0fba9e86799d9ff53dc9541098223 network default NAD default] I0330 15:06:58.367013 3819 helper_linux.go:450] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-58df4cf46c-jplmx, hostIfaceName: b00dafb517eed62, network: default, NAD default, SandboxID: "b00dafb517eed625a2eed981937fc4b4a3c0fba9e86799d9ff53dc9541098223", PCI device ID: , UID: "bf7868ce-c8da-4a3f-909d-64e67cb8fe63", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0330 15:06:58.413075 3819 cni.go:424] [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-jplmx b00dafb517eed625a2eed981937fc4b4a3c0fba9e86799d9ff53dc9541098223 network default NAD default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-58df4cf46c-jplmx b00dafb517eed625a2eed981937fc4b4a3c0fba9e86799d9ff53dc9541098223 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3e:c0:1e:a9:e5:03\",\"name\":\"b00dafb517eed62\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/538dee28-6656-4bbd-9c1a-f9ca69cdb960\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:07:02.079662 3819 namespace.go:98] [cert-manager] adding namespace I0330 15:07:02.080570 3819 namespace.go:102] [cert-manager] adding namespace took 880.51µs I0330 15:07:02.151099 3819 namespace.go:142] [cert-manager] updating namespace I0330 15:07:02.168848 3819 namespace.go:142] [cert-manager] updating namespace I0330 15:07:06.783676 3819 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-fd44f5cbb-x4pc5] creating logical port cert-manager_cert-manager-webhook-fd44f5cbb-x4pc5 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:07:06.783805 3819 kube.go:256] Updating pod cert-manager/cert-manager-webhook-fd44f5cbb-x4pc5 I0330 15:07:06.807149 3819 pod.go:62] [cert-manager/cert-manager-webhook-fd44f5cbb-x4pc5] pod update took 23.379624ms I0330 15:07:06.807176 3819 base_network_controller_pods.go:937] [default/cert-manager/cert-manager-webhook-fd44f5cbb-x4pc5] addLogicalPort annotation time took 23.412932ms I0330 15:07:06.808389 3819 pods.go:271] [cert-manager/cert-manager-webhook-fd44f5cbb-x4pc5] addLogicalPort took 24.733121ms, libovsdb time 875.384µs I0330 15:07:07.157932 3819 cni.go:403] [cert-manager/cert-manager-webhook-fd44f5cbb-x4pc5 44feed7a98c97cea4e01765ddce9a17c24547612259fa4de19e2a8cde35b6e4d network default NAD default] ADD starting CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-x4pc5 44feed7a98c97cea4e01765ddce9a17c24547612259fa4de19e2a8cde35b6e4d network default NAD default] I0330 15:07:07.166597 3819 helper_linux.go:450] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-fd44f5cbb-x4pc5, hostIfaceName: 44feed7a98c97ce, network: default, NAD default, SandboxID: "44feed7a98c97cea4e01765ddce9a17c24547612259fa4de19e2a8cde35b6e4d", PCI device ID: , UID: "57d98466-ca75-4e91-80a8-ef28e4a565dd", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0330 15:07:07.215155 3819 cni.go:424] [cert-manager/cert-manager-webhook-fd44f5cbb-x4pc5 44feed7a98c97cea4e01765ddce9a17c24547612259fa4de19e2a8cde35b6e4d network default NAD default] ADD finished CNI request [cert-manager/cert-manager-webhook-fd44f5cbb-x4pc5 44feed7a98c97cea4e01765ddce9a17c24547612259fa4de19e2a8cde35b6e4d network default NAD default], result "{\"interfaces\":[{\"mac\":\"f6:3d:39:5b:3d:17\",\"name\":\"44feed7a98c97ce\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/417435fa-0b64-40d4-bc7f-310d9c647437\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:07:07.998806 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0330 15:07:09.061334 3819 namespace.go:142] [openshift-operators] updating namespace I0330 15:07:09.083104 3819 namespace.go:142] [openshift-operators] updating namespace W0330 15:07:11.097659 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0330 15:07:11.097685 3819 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0330 15:07:13.050258 3819 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=81 I0330 15:07:13.148068 3819 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk] creating logical port konflux-operator_konflux-operator-controller-manager-7db75c44-92hpk for pod on switch ip-10-0-1-27.ec2.internal I0330 15:07:13.148205 3819 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk I0330 15:07:13.178140 3819 pod.go:62] [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk] pod update took 29.961305ms I0330 15:07:13.178166 3819 base_network_controller_pods.go:937] [default/konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk] addLogicalPort annotation time took 29.992651ms I0330 15:07:13.179383 3819 pods.go:271] [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk] addLogicalPort took 31.331232ms, libovsdb time 883.731µs I0330 15:07:13.506625 3819 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk 04fd4f26269a2482836efee8f54f168f401f34f78f8fbb697408952c99fc62ce network default NAD default] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk 04fd4f26269a2482836efee8f54f168f401f34f78f8fbb697408952c99fc62ce network default NAD default] I0330 15:07:13.516211 3819 helper_linux.go:450] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-7db75c44-92hpk, hostIfaceName: 04fd4f26269a248, network: default, NAD default, SandboxID: "04fd4f26269a2482836efee8f54f168f401f34f78f8fbb697408952c99fc62ce", PCI device ID: , UID: "d66173f4-1aad-48c1-af27-e5b252a38d6d", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0330 15:07:13.562252 3819 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk 04fd4f26269a2482836efee8f54f168f401f34f78f8fbb697408952c99fc62ce network default NAD default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk 04fd4f26269a2482836efee8f54f168f401f34f78f8fbb697408952c99fc62ce network default NAD default], result "{\"interfaces\":[{\"mac\":\"ba:51:89:15:43:78\",\"name\":\"04fd4f26269a248\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/87f6e303-f559-4fcf-b69d-f3ae46ef79cc\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:07:17.087401 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0330 15:07:17.087425 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook W0330 15:07:17.107004 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0330 15:07:17.234365 3819 namespace.go:142] [openshift-operators] updating namespace I0330 15:07:17.621174 3819 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-tntjr] creating logical port openshift-operators_tekton-operator-webhook-668b6fdd5d-tntjr for pod on switch ip-10-0-1-27.ec2.internal I0330 15:07:17.621304 3819 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-668b6fdd5d-tntjr I0330 15:07:17.653848 3819 pod.go:62] [openshift-operators/tekton-operator-webhook-668b6fdd5d-tntjr] pod update took 32.566939ms I0330 15:07:17.653873 3819 base_network_controller_pods.go:937] [default/openshift-operators/tekton-operator-webhook-668b6fdd5d-tntjr] addLogicalPort annotation time took 32.597727ms I0330 15:07:17.655067 3819 pods.go:271] [openshift-operators/tekton-operator-webhook-668b6fdd5d-tntjr] addLogicalPort took 33.916669ms, libovsdb time 861.443µs I0330 15:07:17.989195 3819 cni.go:403] [openshift-operators/tekton-operator-webhook-668b6fdd5d-tntjr a691e6a50ac7ccd31f00ff43bbefcc57f1fe6df781e450ae6308272423b03971 network default NAD default] ADD starting CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-tntjr a691e6a50ac7ccd31f00ff43bbefcc57f1fe6df781e450ae6308272423b03971 network default NAD default] I0330 15:07:17.999693 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-668b6fdd5d-tntjr, hostIfaceName: a691e6a50ac7ccd, network: default, NAD default, SandboxID: "a691e6a50ac7ccd31f00ff43bbefcc57f1fe6df781e450ae6308272423b03971", PCI device ID: , UID: "490945e7-8884-4f08-bb87-4853567f02b4", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0330 15:07:18.052350 3819 cni.go:424] [openshift-operators/tekton-operator-webhook-668b6fdd5d-tntjr a691e6a50ac7ccd31f00ff43bbefcc57f1fe6df781e450ae6308272423b03971 network default NAD default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-668b6fdd5d-tntjr a691e6a50ac7ccd31f00ff43bbefcc57f1fe6df781e450ae6308272423b03971 network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:3f:a0:93:6a:f2\",\"name\":\"a691e6a50ac7ccd\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/18eb12e6-e37a-4fd4-87f5-889993512313\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:07:26.058637 3819 namespace.go:98] [trusted-artifact-signer] adding namespace I0330 15:07:26.059591 3819 namespace.go:102] [trusted-artifact-signer] adding namespace took 889.264µs I0330 15:07:26.165495 3819 namespace.go:142] [trusted-artifact-signer] updating namespace I0330 15:07:26.191807 3819 namespace.go:142] [trusted-artifact-signer] updating namespace W0330 15:07:26.717055 3819 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0330 15:07:27.240691 3819 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-66ccc97d7d-fzqbw] creating logical port trusted-artifact-signer_cli-server-66ccc97d7d-fzqbw for pod on switch ip-10-0-1-27.ec2.internal I0330 15:07:27.240785 3819 kube.go:256] Updating pod trusted-artifact-signer/cli-server-66ccc97d7d-fzqbw I0330 15:07:27.254225 3819 pod.go:62] [trusted-artifact-signer/cli-server-66ccc97d7d-fzqbw] pod update took 13.464754ms I0330 15:07:27.254259 3819 base_network_controller_pods.go:937] [default/trusted-artifact-signer/cli-server-66ccc97d7d-fzqbw] addLogicalPort annotation time took 13.499259ms I0330 15:07:27.255649 3819 pods.go:271] [trusted-artifact-signer/cli-server-66ccc97d7d-fzqbw] addLogicalPort took 14.9749ms, libovsdb time 964.708µs I0330 15:07:27.605704 3819 cni.go:403] [trusted-artifact-signer/cli-server-66ccc97d7d-fzqbw ee94a00aabbd2a3ff22c52d0facf3fa535e2949ca6798b54f14a28cd20e10d30 network default NAD default] ADD starting CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-fzqbw ee94a00aabbd2a3ff22c52d0facf3fa535e2949ca6798b54f14a28cd20e10d30 network default NAD default] I0330 15:07:27.614362 3819 helper_linux.go:450] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-66ccc97d7d-fzqbw, hostIfaceName: ee94a00aabbd2a3, network: default, NAD default, SandboxID: "ee94a00aabbd2a3ff22c52d0facf3fa535e2949ca6798b54f14a28cd20e10d30", PCI device ID: , UID: "40f50195-d72e-4c7c-bb06-e1d927efcda2", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0330 15:07:27.866786 3819 cni.go:424] [trusted-artifact-signer/cli-server-66ccc97d7d-fzqbw ee94a00aabbd2a3ff22c52d0facf3fa535e2949ca6798b54f14a28cd20e10d30 network default NAD default] ADD finished CNI request [trusted-artifact-signer/cli-server-66ccc97d7d-fzqbw ee94a00aabbd2a3ff22c52d0facf3fa535e2949ca6798b54f14a28cd20e10d30 network default NAD default], result "{\"interfaces\":[{\"mac\":\"82:ca:f3:0e:3c:48\",\"name\":\"ee94a00aabbd2a3\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/54e4cc75-05f7-4afe-a6ae-d1d16a92621f\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:07:28.105639 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0330 15:07:29.537178 3819 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:07:29.537840 3819 obj_retry.go:464] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:07:29.575331 3819 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:07:29.575359 3819 obj_retry.go:458] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:07:29.584084 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0330 15:07:29.584135 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0330 15:07:32.737615 3819 namespace.go:98] [openshift-pipelines] adding namespace I0330 15:07:32.738596 3819 namespace.go:102] [openshift-pipelines] adding namespace took 952.9µs I0330 15:07:32.749971 3819 namespace.go:142] [openshift-pipelines] updating namespace I0330 15:07:33.435775 3819 namespace.go:142] [cert-manager] updating namespace I0330 15:07:33.638605 3819 namespace.go:142] [cert-manager-operator] updating namespace I0330 15:07:34.040973 3819 namespace.go:142] [konflux-operator] updating namespace I0330 15:07:34.245588 3819 namespace.go:142] [konflux-ui] updating namespace I0330 15:07:34.443418 3819 namespace.go:142] [open-cluster-management-2pc3srqfa75va1b7r65hvfp4e15g46er] updating namespace I0330 15:07:34.845161 3819 namespace.go:142] [rhbk-operator] updating namespace I0330 15:07:35.048274 3819 namespace.go:142] [rhtpa-operator] updating namespace I0330 15:07:35.248163 3819 namespace.go:142] [trusted-artifact-signer] updating namespace I0330 15:07:35.447883 3819 namespace.go:142] [tsf] updating namespace I0330 15:07:35.655680 3819 namespace.go:142] [tsf-keycloak] updating namespace I0330 15:07:35.850267 3819 namespace.go:142] [tsf-tas] updating namespace I0330 15:07:36.051765 3819 namespace.go:142] [tsf-tpa] updating namespace I0330 15:07:36.180311 3819 namespace.go:142] [cert-manager] updating namespace I0330 15:07:36.191163 3819 namespace.go:142] [cert-manager-operator] updating namespace I0330 15:07:36.198118 3819 namespace.go:142] [default] updating namespace I0330 15:07:36.207519 3819 namespace.go:142] [konflux-operator] updating namespace I0330 15:07:36.216429 3819 namespace.go:142] [konflux-ui] updating namespace I0330 15:07:36.229828 3819 namespace.go:142] [open-cluster-management-2pc3srqfa75va1b7r65hvfp4e15g46er] updating namespace I0330 15:07:36.235991 3819 namespace.go:142] [openshift] updating namespace I0330 15:07:36.244885 3819 namespace.go:142] [rhbk-operator] updating namespace I0330 15:07:36.254487 3819 namespace.go:142] [rhtpa-operator] updating namespace I0330 15:07:36.264372 3819 namespace.go:142] [trusted-artifact-signer] updating namespace I0330 15:07:36.273611 3819 namespace.go:142] [tsf] updating namespace I0330 15:07:36.296114 3819 namespace.go:142] [tsf-keycloak] updating namespace I0330 15:07:36.312907 3819 namespace.go:142] [tsf-tas] updating namespace I0330 15:07:36.334484 3819 namespace.go:142] [tsf-tpa] updating namespace I0330 15:07:36.436247 3819 namespace.go:142] [cert-manager] updating namespace I0330 15:07:36.533665 3819 namespace.go:142] [cert-manager-operator] updating namespace I0330 15:07:36.633994 3819 namespace.go:142] [default] updating namespace I0330 15:07:36.739337 3819 namespace.go:142] [konflux-operator] updating namespace I0330 15:07:36.837156 3819 namespace.go:142] [konflux-ui] updating namespace I0330 15:07:36.936496 3819 namespace.go:142] [open-cluster-management-2pc3srqfa75va1b7r65hvfp4e15g46er] updating namespace I0330 15:07:37.034468 3819 namespace.go:142] [openshift] updating namespace I0330 15:07:37.139727 3819 namespace.go:142] [rhbk-operator] updating namespace I0330 15:07:37.266165 3819 namespace.go:142] [rhtpa-operator] updating namespace I0330 15:07:37.354184 3819 namespace.go:142] [trusted-artifact-signer] updating namespace I0330 15:07:37.439763 3819 namespace.go:142] [tsf] updating namespace I0330 15:07:37.541399 3819 namespace.go:142] [tsf-keycloak] updating namespace I0330 15:07:37.644558 3819 namespace.go:142] [tsf-tas] updating namespace I0330 15:07:37.755956 3819 namespace.go:142] [tsf-tpa] updating namespace I0330 15:07:38.261705 3819 namespace.go:142] [openshift-pipelines] updating namespace W0330 15:07:43.445624 3819 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0330 15:07:44.719231 3819 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0330 15:07:45.893515 3819 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-b79bdd59-fv6t9] creating logical port openshift-pipelines_tekton-events-controller-b79bdd59-fv6t9 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:07:45.893668 3819 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-b79bdd59-fv6t9 I0330 15:07:45.910178 3819 pod.go:62] [openshift-pipelines/tekton-events-controller-b79bdd59-fv6t9] pod update took 16.53947ms I0330 15:07:45.910205 3819 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-events-controller-b79bdd59-fv6t9] addLogicalPort annotation time took 16.572414ms I0330 15:07:45.911590 3819 pods.go:271] [openshift-pipelines/tekton-events-controller-b79bdd59-fv6t9] addLogicalPort took 18.089342ms, libovsdb time 1.010171ms I0330 15:07:46.091109 3819 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-8f55v] creating logical port openshift-pipelines_tekton-pipelines-webhook-584c448755-8f55v for pod on switch ip-10-0-1-27.ec2.internal I0330 15:07:46.091214 3819 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-584c448755-8f55v I0330 15:07:46.115876 3819 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-584c448755-8f55v] pod update took 24.696589ms I0330 15:07:46.115907 3819 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-pipelines-webhook-584c448755-8f55v] addLogicalPort annotation time took 24.7301ms I0330 15:07:46.117121 3819 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-584c448755-8f55v] addLogicalPort took 26.034551ms, libovsdb time 869.19µs I0330 15:07:46.186682 3819 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-p9krs] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-7988b548df-p9krs for pod on switch ip-10-0-1-27.ec2.internal I0330 15:07:46.186777 3819 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-p9krs I0330 15:07:46.203102 3819 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-p9krs] pod update took 16.346921ms I0330 15:07:46.203134 3819 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-p9krs] addLogicalPort annotation time took 16.385391ms I0330 15:07:46.204427 3819 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-p9krs] addLogicalPort took 17.759922ms, libovsdb time 891.55µs I0330 15:07:46.259911 3819 cni.go:403] [openshift-pipelines/tekton-events-controller-b79bdd59-fv6t9 41a184f8ae0e081aaaabaf05b751b8d28a0d01b01a7750542d36c3da6c261498 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-fv6t9 41a184f8ae0e081aaaabaf05b751b8d28a0d01b01a7750542d36c3da6c261498 network default NAD default] I0330 15:07:46.268161 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-b79bdd59-fv6t9, hostIfaceName: 41a184f8ae0e081, network: default, NAD default, SandboxID: "41a184f8ae0e081aaaabaf05b751b8d28a0d01b01a7750542d36c3da6c261498", PCI device ID: , UID: "4177335c-c803-4a52-92cb-158cfae1488b", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0330 15:07:46.314607 3819 cni.go:424] [openshift-pipelines/tekton-events-controller-b79bdd59-fv6t9 41a184f8ae0e081aaaabaf05b751b8d28a0d01b01a7750542d36c3da6c261498 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-b79bdd59-fv6t9 41a184f8ae0e081aaaabaf05b751b8d28a0d01b01a7750542d36c3da6c261498 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:bd:3a:ab:f3:7e\",\"name\":\"41a184f8ae0e081\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5255b21a-21c5-4933-bc7f-a0ba2f1e7695\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:07:46.455722 3819 cni.go:403] [openshift-pipelines/tekton-pipelines-webhook-584c448755-8f55v e48516f426bfe25630d986454486a74f3de37d5835b22abf796f5ad254fb0eea network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-8f55v e48516f426bfe25630d986454486a74f3de37d5835b22abf796f5ad254fb0eea network default NAD default] I0330 15:07:46.464279 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-584c448755-8f55v, hostIfaceName: e48516f426bfe25, network: default, NAD default, SandboxID: "e48516f426bfe25630d986454486a74f3de37d5835b22abf796f5ad254fb0eea", PCI device ID: , UID: "28c12c04-0a39-4e61-937f-4ccfc12cbf8d", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0330 15:07:46.510402 3819 cni.go:424] [openshift-pipelines/tekton-pipelines-webhook-584c448755-8f55v e48516f426bfe25630d986454486a74f3de37d5835b22abf796f5ad254fb0eea network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-584c448755-8f55v e48516f426bfe25630d986454486a74f3de37d5835b22abf796f5ad254fb0eea network default NAD default], result "{\"interfaces\":[{\"mac\":\"02:f6:59:32:c6:7d\",\"name\":\"e48516f426bfe25\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1a31e449-9341-45d9-adc8-c34dc69b821e\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:07:46.556120 3819 cni.go:403] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-p9krs 280b76557ed0fc36fe6aafb172d039c4b22e69e0f6a23051d0bcfcb7d2a689cd network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-p9krs 280b76557ed0fc36fe6aafb172d039c4b22e69e0f6a23051d0bcfcb7d2a689cd network default NAD default] I0330 15:07:46.565000 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-7988b548df-p9krs, hostIfaceName: 280b76557ed0fc3, network: default, NAD default, SandboxID: "280b76557ed0fc36fe6aafb172d039c4b22e69e0f6a23051d0bcfcb7d2a689cd", PCI device ID: , UID: "ce8af281-0040-4797-ba76-3044db7caa22", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0330 15:07:46.612477 3819 cni.go:424] [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-p9krs 280b76557ed0fc36fe6aafb172d039c4b22e69e0f6a23051d0bcfcb7d2a689cd network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-7988b548df-p9krs 280b76557ed0fc36fe6aafb172d039c4b22e69e0f6a23051d0bcfcb7d2a689cd network default NAD default], result "{\"interfaces\":[{\"mac\":\"26:2a:38:1b:2c:4d\",\"name\":\"280b76557ed0fc3\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2ee7f8ab-33b9-495f-8fc4-a62a4d3766f3\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:07:50.313458 3819 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0330 15:07:50.313481 3819 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0330 15:07:50.313490 3819 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0330 15:07:50.313496 3819 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0330 15:07:50.338160 3819 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0330 15:07:50.338181 3819 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0330 15:07:50.338189 3819 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0330 15:07:50.514857 3819 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0330 15:07:50.514892 3819 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0330 15:07:50.514899 3819 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0330 15:07:55.156691 3819 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 W0330 15:08:01.300274 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0330 15:08:01.300304 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0330 15:08:01.300311 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0330 15:08:01.327127 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0330 15:08:01.327151 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0330 15:08:01.327158 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0330 15:08:01.327163 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0330 15:08:04.042913 3819 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=105 I0330 15:08:08.692862 3819 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:08:08.693433 3819 obj_retry.go:464] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:08:08.719801 3819 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:08:08.719834 3819 obj_retry.go:458] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:08:08.721100 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0330 15:08:08.721119 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure W0330 15:08:12.641465 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0330 15:08:16.612217 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0330 15:08:16.612237 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0330 15:08:16.612242 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0330 15:08:21.147424 3819 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0330 15:08:25.441492 3819 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-cxpdw] creating logical port openshift-pipelines_tekton-chains-controller-5fb999ffff-cxpdw for pod on switch ip-10-0-1-27.ec2.internal I0330 15:08:25.441627 3819 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-5fb999ffff-cxpdw I0330 15:08:25.455072 3819 pod.go:62] [openshift-pipelines/tekton-chains-controller-5fb999ffff-cxpdw] pod update took 13.478444ms I0330 15:08:25.455100 3819 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-chains-controller-5fb999ffff-cxpdw] addLogicalPort annotation time took 13.51128ms I0330 15:08:25.456376 3819 pods.go:271] [openshift-pipelines/tekton-chains-controller-5fb999ffff-cxpdw] addLogicalPort took 14.904207ms, libovsdb time 907.73µs I0330 15:08:25.813680 3819 cni.go:403] [openshift-pipelines/tekton-chains-controller-5fb999ffff-cxpdw 55951137dc9b8d7b67f19e7dc47d9a45c6dceb31da897b667a4f26d3b98b0b61 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-cxpdw 55951137dc9b8d7b67f19e7dc47d9a45c6dceb31da897b667a4f26d3b98b0b61 network default NAD default] I0330 15:08:25.822367 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-5fb999ffff-cxpdw, hostIfaceName: 55951137dc9b8d7, network: default, NAD default, SandboxID: "55951137dc9b8d7b67f19e7dc47d9a45c6dceb31da897b667a4f26d3b98b0b61", PCI device ID: , UID: "fff09c1e-c8be-453f-80d3-4e284ba5ccc3", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0330 15:08:25.870153 3819 cni.go:424] [openshift-pipelines/tekton-chains-controller-5fb999ffff-cxpdw 55951137dc9b8d7b67f19e7dc47d9a45c6dceb31da897b667a4f26d3b98b0b61 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-5fb999ffff-cxpdw 55951137dc9b8d7b67f19e7dc47d9a45c6dceb31da897b667a4f26d3b98b0b61 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8e:75:05:50:d3:9e\",\"name\":\"55951137dc9b8d7\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c8167323-c505-476b-9bc7-7a13076ad7ae\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:08:28.053214 3819 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=740 I0330 15:08:30.376375 3819 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0330 15:08:33.448774 3819 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:08:33.448870 3819 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0330 15:08:33.463855 3819 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 15.008141ms I0330 15:08:33.463883 3819 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 15.044739ms I0330 15:08:33.465553 3819 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 16.779062ms, libovsdb time 1.231066ms I0330 15:08:41.614804 3819 cni.go:403] [openshift-pipelines/tekton-results-postgres-0 972695bfabd85034d6985932223c74dc5ac00ae114712da95de5bce5d51c74fd network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 972695bfabd85034d6985932223c74dc5ac00ae114712da95de5bce5d51c74fd network default NAD default] I0330 15:08:41.623454 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 972695bfabd8503, network: default, NAD default, SandboxID: "972695bfabd85034d6985932223c74dc5ac00ae114712da95de5bce5d51c74fd", PCI device ID: , UID: "628fc91c-bac0-4226-b6c0-e7d5176e16ae", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0330 15:08:41.671706 3819 cni.go:424] [openshift-pipelines/tekton-results-postgres-0 972695bfabd85034d6985932223c74dc5ac00ae114712da95de5bce5d51c74fd network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 972695bfabd85034d6985932223c74dc5ac00ae114712da95de5bce5d51c74fd network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:0a:cf:34:95:6e\",\"name\":\"972695bfabd8503\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f29e51b4-559e-4b5a-8b57-98a1d85db685\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:08:43.899406 3819 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:08:43.900019 3819 obj_retry.go:464] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:08:43.938911 3819 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:08:43.938939 3819 obj_retry.go:458] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:08:43.944573 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0330 15:08:43.944606 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0330 15:08:45.600800 3819 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-xh6cd-k6hg7] creating logical port tsf-tas_segment-backup-installation-xh6cd-k6hg7 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:08:45.600889 3819 kube.go:256] Updating pod tsf-tas/segment-backup-installation-xh6cd-k6hg7 I0330 15:08:45.614042 3819 pod.go:62] [tsf-tas/segment-backup-installation-xh6cd-k6hg7] pod update took 13.174015ms I0330 15:08:45.614081 3819 base_network_controller_pods.go:937] [default/tsf-tas/segment-backup-installation-xh6cd-k6hg7] addLogicalPort annotation time took 13.202832ms I0330 15:08:45.615341 3819 pods.go:271] [tsf-tas/segment-backup-installation-xh6cd-k6hg7] addLogicalPort took 14.558665ms, libovsdb time 869.255µs I0330 15:08:46.168425 3819 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0330 15:08:46.689887 3819 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-5cbbf5fb5b-jbwxl] creating logical port tsf-tas_trillian-logserver-5cbbf5fb5b-jbwxl for pod on switch ip-10-0-1-27.ec2.internal I0330 15:08:46.690005 3819 kube.go:256] Updating pod tsf-tas/trillian-logserver-5cbbf5fb5b-jbwxl I0330 15:08:46.707703 3819 pod.go:62] [tsf-tas/trillian-logserver-5cbbf5fb5b-jbwxl] pod update took 17.730886ms I0330 15:08:46.707738 3819 base_network_controller_pods.go:937] [default/tsf-tas/trillian-logserver-5cbbf5fb5b-jbwxl] addLogicalPort annotation time took 17.771182ms I0330 15:08:46.709334 3819 pods.go:271] [tsf-tas/trillian-logserver-5cbbf5fb5b-jbwxl] addLogicalPort took 19.466223ms, libovsdb time 1.074225ms I0330 15:08:46.845716 3819 cni.go:403] [tsf-tas/segment-backup-installation-xh6cd-k6hg7 2effbb07844279f63110b9a04151e3a2814c5bd18c17c00973a8296365aa78f2 network default NAD default] ADD starting CNI request [tsf-tas/segment-backup-installation-xh6cd-k6hg7 2effbb07844279f63110b9a04151e3a2814c5bd18c17c00973a8296365aa78f2 network default NAD default] I0330 15:08:46.855014 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-xh6cd-k6hg7, hostIfaceName: 2effbb07844279f, network: default, NAD default, SandboxID: "2effbb07844279f63110b9a04151e3a2814c5bd18c17c00973a8296365aa78f2", PCI device ID: , UID: "d8a9d4bd-f414-4888-8969-76df92930609", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0330 15:08:46.904629 3819 cni.go:424] [tsf-tas/segment-backup-installation-xh6cd-k6hg7 2effbb07844279f63110b9a04151e3a2814c5bd18c17c00973a8296365aa78f2 network default NAD default] ADD finished CNI request [tsf-tas/segment-backup-installation-xh6cd-k6hg7 2effbb07844279f63110b9a04151e3a2814c5bd18c17c00973a8296365aa78f2 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6a:c4:ff:8f:bb:e7\",\"name\":\"2effbb07844279f\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7dbdc34c-610f-4b2a-bf6c-e4fe2221a828\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:08:47.068813 3819 cni.go:403] [tsf-tas/trillian-logserver-5cbbf5fb5b-jbwxl 94e016cfe70648bd4787b71dfc86680ba3ac97c210aad4144cbe06ce3f0d9bf1 network default NAD default] ADD starting CNI request [tsf-tas/trillian-logserver-5cbbf5fb5b-jbwxl 94e016cfe70648bd4787b71dfc86680ba3ac97c210aad4144cbe06ce3f0d9bf1 network default NAD default] I0330 15:08:47.078823 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-5cbbf5fb5b-jbwxl, hostIfaceName: 94e016cfe70648b, network: default, NAD default, SandboxID: "94e016cfe70648bd4787b71dfc86680ba3ac97c210aad4144cbe06ce3f0d9bf1", PCI device ID: , UID: "1fe10b4c-2397-4cb9-9b56-61177c2105e0", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0330 15:08:47.094800 3819 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-2wtgp-6869j] creating logical port tsf-tas_rekor-createtree-job-2wtgp-6869j for pod on switch ip-10-0-1-27.ec2.internal I0330 15:08:47.094940 3819 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-2wtgp-6869j I0330 15:08:47.115650 3819 pod.go:62] [tsf-tas/rekor-createtree-job-2wtgp-6869j] pod update took 20.764883ms I0330 15:08:47.115685 3819 base_network_controller_pods.go:937] [default/tsf-tas/rekor-createtree-job-2wtgp-6869j] addLogicalPort annotation time took 20.807799ms I0330 15:08:47.117318 3819 pods.go:271] [tsf-tas/rekor-createtree-job-2wtgp-6869j] addLogicalPort took 22.549721ms, libovsdb time 927.329µs I0330 15:08:47.335718 3819 cni.go:424] [tsf-tas/trillian-logserver-5cbbf5fb5b-jbwxl 94e016cfe70648bd4787b71dfc86680ba3ac97c210aad4144cbe06ce3f0d9bf1 network default NAD default] ADD finished CNI request [tsf-tas/trillian-logserver-5cbbf5fb5b-jbwxl 94e016cfe70648bd4787b71dfc86680ba3ac97c210aad4144cbe06ce3f0d9bf1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"6e:82:80:df:e4:f9\",\"name\":\"94e016cfe70648b\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/faa4aa33-761e-486c-acd1-19248068a14b\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:08:47.459864 3819 cni.go:403] [tsf-tas/rekor-createtree-job-2wtgp-6869j 5492f31168f0053897d07489dc871faf6d6dbace887b3a7a4b0a9fb375347b7e network default NAD default] ADD starting CNI request [tsf-tas/rekor-createtree-job-2wtgp-6869j 5492f31168f0053897d07489dc871faf6d6dbace887b3a7a4b0a9fb375347b7e network default NAD default] I0330 15:08:47.488575 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-2wtgp-6869j, hostIfaceName: 5492f31168f0053, network: default, NAD default, SandboxID: "5492f31168f0053897d07489dc871faf6d6dbace887b3a7a4b0a9fb375347b7e", PCI device ID: , UID: "d55b7d61-cb70-4f2e-8fa7-4ae2f3beb1b6", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0330 15:08:47.536837 3819 cni.go:424] [tsf-tas/rekor-createtree-job-2wtgp-6869j 5492f31168f0053897d07489dc871faf6d6dbace887b3a7a4b0a9fb375347b7e network default NAD default] ADD finished CNI request [tsf-tas/rekor-createtree-job-2wtgp-6869j 5492f31168f0053897d07489dc871faf6d6dbace887b3a7a4b0a9fb375347b7e network default NAD default], result "{\"interfaces\":[{\"mac\":\"52:a5:f2:df:4d:4f\",\"name\":\"5492f31168f0053\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0135e5d8-33ea-442d-b3ef-536dcfc80798\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:08:47.657560 3819 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0330 15:08:49.043493 3819 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0330 15:08:49.043525 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server I0330 15:08:49.083697 3819 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-vwtlj] creating logical port openshift-pipelines_tekton-results-retention-policy-agent-55cbb85cb8-vwtlj for pod on switch ip-10-0-1-27.ec2.internal I0330 15:08:49.083892 3819 kube.go:256] Updating pod openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-vwtlj I0330 15:08:49.099348 3819 pod.go:62] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-vwtlj] pod update took 15.491132ms I0330 15:08:49.099449 3819 base_network_controller_pods.go:937] [default/openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-vwtlj] addLogicalPort annotation time took 15.596825ms I0330 15:08:49.101286 3819 pods.go:271] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-vwtlj] addLogicalPort took 17.612132ms, libovsdb time 1.237775ms W0330 15:08:54.073158 3819 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0330 15:08:56.042316 3819 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0330 15:08:58.042830 3819 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=195 W0330 15:08:58.100414 3819 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0330 15:08:58.100434 3819 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0330 15:08:58.100439 3819 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0330 15:08:59.964379 3819 cni.go:403] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-vwtlj 7b768c5c31745b98b9fe47a05cbd1f0e812359d79fdfac1586657f5dd0e47a25 network default NAD default] ADD starting CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-vwtlj 7b768c5c31745b98b9fe47a05cbd1f0e812359d79fdfac1586657f5dd0e47a25 network default NAD default] I0330 15:08:59.973095 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-retention-policy-agent-55cbb85cb8-vwtlj, hostIfaceName: 7b768c5c31745b9, network: default, NAD default, SandboxID: "7b768c5c31745b98b9fe47a05cbd1f0e812359d79fdfac1586657f5dd0e47a25", PCI device ID: , UID: "08d35580-4379-44f2-991d-2c900312e6c9", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0330 15:08:59.999452 3819 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-z9c8m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:00.000019 3819 obj_retry.go:464] Detected object tsf-keycloak/tsf-iam-z9c8m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:00.031803 3819 cni.go:424] [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-vwtlj 7b768c5c31745b98b9fe47a05cbd1f0e812359d79fdfac1586657f5dd0e47a25 network default NAD default] ADD finished CNI request [openshift-pipelines/tekton-results-retention-policy-agent-55cbb85cb8-vwtlj 7b768c5c31745b98b9fe47a05cbd1f0e812359d79fdfac1586657f5dd0e47a25 network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:f2:a5:65:44:51\",\"name\":\"7b768c5c31745b9\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85ec01ef-d20c-448e-87c8-9e345b83853b\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:09:01.024162 3819 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-z9c8m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:01.024194 3819 obj_retry.go:458] Detected object tsf-keycloak/tsf-iam-z9c8m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0330 15:09:01.121130 3819 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0330 15:09:01.362283 3819 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:01.362871 3819 obj_retry.go:464] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0330 15:09:01.730392 3819 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0330 15:09:01.872776 3819 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:01.872805 3819 obj_retry.go:458] Detected object tsf-keycloak/keycloak-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:01.879251 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-keycloak/keycloak-0 I0330 15:09:01.879293 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-keycloak/keycloak-0 W0330 15:09:03.901623 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0330 15:09:03.901649 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0330 15:09:03.901652 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0330 15:09:04.734136 3819 cni.go:403] [tsf-tas/segment-backup-installation-xh6cd-k6hg7 2effbb07844279f63110b9a04151e3a2814c5bd18c17c00973a8296365aa78f2 network default NAD default] DEL starting CNI request [tsf-tas/segment-backup-installation-xh6cd-k6hg7 2effbb07844279f63110b9a04151e3a2814c5bd18c17c00973a8296365aa78f2 network default NAD default] I0330 15:09:04.828519 3819 cni.go:424] [tsf-tas/segment-backup-installation-xh6cd-k6hg7 2effbb07844279f63110b9a04151e3a2814c5bd18c17c00973a8296365aa78f2 network default NAD default] DEL finished CNI request [tsf-tas/segment-backup-installation-xh6cd-k6hg7 2effbb07844279f63110b9a04151e3a2814c5bd18c17c00973a8296365aa78f2 network default NAD default], result "{}", err I0330 15:09:04.880074 3819 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-xh6cd-k6hg7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:04.880110 3819 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-xh6cd-k6hg7 I0330 15:09:04.881486 3819 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-xh6cd-k6hg7, ips: 10.129.0.41 I0330 15:09:04.881524 3819 obj_retry.go:464] Detected object tsf-tas/segment-backup-installation-xh6cd-k6hg7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:05.769887 3819 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-xh6cd-k6hg7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:05.769916 3819 obj_retry.go:458] Detected object tsf-tas/segment-backup-installation-xh6cd-k6hg7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0330 15:09:29.827119 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0330 15:09:34.412947 3819 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-fhd52-6hcb6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:34.413567 3819 obj_retry.go:464] Detected object tsf-tas/ctlog-createtree-job-fhd52-6hcb6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:35.212688 3819 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:35.213283 3819 obj_retry.go:464] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:35.235261 3819 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:35.235287 3819 obj_retry.go:458] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:35.244014 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0330 15:09:35.244043 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0330 15:09:35.311061 3819 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-fhd52-6hcb6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:35.311097 3819 obj_retry.go:458] Detected object tsf-tas/ctlog-createtree-job-fhd52-6hcb6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:36.816360 3819 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-fb5cdb5b5-6xjkx] creating logical port tsf-tas_ctlog-fb5cdb5b5-6xjkx for pod on switch ip-10-0-1-27.ec2.internal I0330 15:09:36.816452 3819 kube.go:256] Updating pod tsf-tas/ctlog-fb5cdb5b5-6xjkx I0330 15:09:36.832810 3819 pod.go:62] [tsf-tas/ctlog-fb5cdb5b5-6xjkx] pod update took 16.38187ms I0330 15:09:36.832845 3819 base_network_controller_pods.go:937] [default/tsf-tas/ctlog-fb5cdb5b5-6xjkx] addLogicalPort annotation time took 16.423176ms I0330 15:09:36.834263 3819 pods.go:271] [tsf-tas/ctlog-fb5cdb5b5-6xjkx] addLogicalPort took 17.923183ms, libovsdb time 872.338µs I0330 15:09:37.203476 3819 cni.go:403] [tsf-tas/ctlog-fb5cdb5b5-6xjkx fae733d10b417b1ca0395d20d07d9f514842bc6f0e89f7a6882fa7c8e99c8b06 network default NAD default] ADD starting CNI request [tsf-tas/ctlog-fb5cdb5b5-6xjkx fae733d10b417b1ca0395d20d07d9f514842bc6f0e89f7a6882fa7c8e99c8b06 network default NAD default] I0330 15:09:37.212317 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: ctlog-fb5cdb5b5-6xjkx, hostIfaceName: fae733d10b417b1, network: default, NAD default, SandboxID: "fae733d10b417b1ca0395d20d07d9f514842bc6f0e89f7a6882fa7c8e99c8b06", PCI device ID: , UID: "3f522afc-3acb-481e-9a6a-69d112ec50c4", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0330 15:09:37.263988 3819 cni.go:424] [tsf-tas/ctlog-fb5cdb5b5-6xjkx fae733d10b417b1ca0395d20d07d9f514842bc6f0e89f7a6882fa7c8e99c8b06 network default NAD default] ADD finished CNI request [tsf-tas/ctlog-fb5cdb5b5-6xjkx fae733d10b417b1ca0395d20d07d9f514842bc6f0e89f7a6882fa7c8e99c8b06 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7a:82:ac:aa:95:f2\",\"name\":\"fae733d10b417b1\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3b661660-32a8-4c94-ab90-5392132c9fdd\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:09:38.505577 3819 namespace.go:142] [openshift-pipelines] updating namespace W0330 15:09:39.937454 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog W0330 15:09:42.119482 3819 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0330 15:09:42.140561 3819 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0330 15:09:42.140582 3819 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller I0330 15:09:44.476491 3819 obj_retry.go:464] Detected object tsf/patch-tekton-config-zszp8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:44.477097 3819 obj_retry.go:464] Detected object tsf/patch-tekton-config-zszp8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:45.358364 3819 obj_retry.go:458] Detected object tsf/patch-tekton-config-zszp8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:45.358393 3819 obj_retry.go:458] Detected object tsf/patch-tekton-config-zszp8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0330 15:09:45.576966 3819 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0330 15:09:50.958671 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog I0330 15:09:52.522988 3819 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-7qwlh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:52.523589 3819 obj_retry.go:464] Detected object tsf/tsf-tekton-configuration-7qwlh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:53.543014 3819 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-7qwlh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:53.543046 3819 obj_retry.go:458] Detected object tsf/tsf-tekton-configuration-7qwlh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:53.582910 3819 obj_retry.go:458] Detected object tsf/patch-tekton-config-zszp8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:53.582938 3819 obj_retry.go:458] Detected object tsf/patch-tekton-config-zszp8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:09:53.592051 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-zszp8 I0330 15:09:53.592084 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-zszp8 I0330 15:09:54.083057 3819 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-mrhj7] creating logical port openshift-pipelines_pipelines-console-plugin-79cb8f5447-mrhj7 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:09:54.083162 3819 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-79cb8f5447-mrhj7 I0330 15:09:54.098491 3819 pod.go:62] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-mrhj7] pod update took 15.358725ms I0330 15:09:54.098525 3819 base_network_controller_pods.go:937] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-mrhj7] addLogicalPort annotation time took 15.400902ms I0330 15:09:54.100067 3819 pods.go:271] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-mrhj7] addLogicalPort took 17.028434ms, libovsdb time 1.093802ms I0330 15:09:54.452752 3819 cni.go:403] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-mrhj7 a2d86c3010634fdca05becb1c5f7455bc5436b80d8fd9350cca243751f88105e network default NAD default] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-mrhj7 a2d86c3010634fdca05becb1c5f7455bc5436b80d8fd9350cca243751f88105e network default NAD default] I0330 15:09:54.461416 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-79cb8f5447-mrhj7, hostIfaceName: a2d86c3010634fd, network: default, NAD default, SandboxID: "a2d86c3010634fdca05becb1c5f7455bc5436b80d8fd9350cca243751f88105e", PCI device ID: , UID: "7c173b00-4a03-40c6-99ec-48f2428b512f", MAC: 0a:58:0a:81:00:2e, IPs: [10.129.0.46/23] I0330 15:09:54.719827 3819 cni.go:424] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-mrhj7 a2d86c3010634fdca05becb1c5f7455bc5436b80d8fd9350cca243751f88105e network default NAD default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-mrhj7 a2d86c3010634fdca05becb1c5f7455bc5436b80d8fd9350cca243751f88105e network default NAD default], result "{\"interfaces\":[{\"mac\":\"86:6c:9a:ca:de:3d\",\"name\":\"a2d86c3010634fd\"},{\"mac\":\"0a:58:0a:81:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0b3c8563-204a-41eb-a095-7f1b2318225c\"}],\"ips\":[{\"address\":\"10.129.0.46/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:09:59.057926 3819 cni.go:403] [tsf-tas/rekor-createtree-job-2wtgp-6869j 5492f31168f0053897d07489dc871faf6d6dbace887b3a7a4b0a9fb375347b7e network default NAD default] DEL starting CNI request [tsf-tas/rekor-createtree-job-2wtgp-6869j 5492f31168f0053897d07489dc871faf6d6dbace887b3a7a4b0a9fb375347b7e network default NAD default] I0330 15:09:59.163803 3819 cni.go:424] [tsf-tas/rekor-createtree-job-2wtgp-6869j 5492f31168f0053897d07489dc871faf6d6dbace887b3a7a4b0a9fb375347b7e network default NAD default] DEL finished CNI request [tsf-tas/rekor-createtree-job-2wtgp-6869j 5492f31168f0053897d07489dc871faf6d6dbace887b3a7a4b0a9fb375347b7e network default NAD default], result "{}", err I0330 15:09:59.217880 3819 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-2wtgp-6869j of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:09:59.217917 3819 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-2wtgp-6869j I0330 15:09:59.219630 3819 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-2wtgp-6869j, ips: 10.129.0.43 I0330 15:09:59.219667 3819 obj_retry.go:464] Detected object tsf-tas/rekor-createtree-job-2wtgp-6869j of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:00.063291 3819 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-2wtgp-6869j of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:00.063324 3819 obj_retry.go:458] Detected object tsf-tas/rekor-createtree-job-2wtgp-6869j of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:00.167941 3819 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29581390-sghjm for pod on switch ip-10-0-1-27.ec2.internal I0330 15:10:00.168136 3819 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm I0330 15:10:00.180480 3819 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm] pod update took 12.376101ms I0330 15:10:00.180513 3819 base_network_controller_pods.go:937] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm] addLogicalPort annotation time took 12.415724ms I0330 15:10:00.181937 3819 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm] addLogicalPort took 14.015203ms, libovsdb time 950.455µs I0330 15:10:01.711788 3819 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm 43ea94a526c403690a366f197e5b4aa2e69d5295646c29834e34665d52b6686e network default NAD default] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm 43ea94a526c403690a366f197e5b4aa2e69d5295646c29834e34665d52b6686e network default NAD default] I0330 15:10:01.720229 3819 helper_linux.go:450] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29581390-sghjm, hostIfaceName: 43ea94a526c4036, network: default, NAD default, SandboxID: "43ea94a526c403690a366f197e5b4aa2e69d5295646c29834e34665d52b6686e", PCI device ID: , UID: "a1e3b8ce-cb31-46d2-86ee-25206942e2e8", MAC: 0a:58:0a:81:00:2f, IPs: [10.129.0.47/23] I0330 15:10:01.770322 3819 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm 43ea94a526c403690a366f197e5b4aa2e69d5295646c29834e34665d52b6686e network default NAD default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm 43ea94a526c403690a366f197e5b4aa2e69d5295646c29834e34665d52b6686e network default NAD default], result "{\"interfaces\":[{\"mac\":\"e6:dd:6e:c5:37:58\",\"name\":\"43ea94a526c4036\"},{\"mac\":\"0a:58:0a:81:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/33eea4da-f04a-4216-8b3b-edb96b598d8a\"}],\"ips\":[{\"address\":\"10.129.0.47/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:10:03.327360 3819 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-7cf5f954b6-5ww4l] creating logical port tsf-tas_rekor-redis-7cf5f954b6-5ww4l for pod on switch ip-10-0-1-27.ec2.internal I0330 15:10:03.327462 3819 kube.go:256] Updating pod tsf-tas/rekor-redis-7cf5f954b6-5ww4l I0330 15:10:03.340044 3819 pod.go:62] [tsf-tas/rekor-redis-7cf5f954b6-5ww4l] pod update took 12.603472ms I0330 15:10:03.340076 3819 base_network_controller_pods.go:937] [default/tsf-tas/rekor-redis-7cf5f954b6-5ww4l] addLogicalPort annotation time took 12.642344ms I0330 15:10:03.341525 3819 pods.go:271] [tsf-tas/rekor-redis-7cf5f954b6-5ww4l] addLogicalPort took 14.187325ms, libovsdb time 905.396µs I0330 15:10:04.294493 3819 cni.go:403] [tsf-tas/rekor-redis-7cf5f954b6-5ww4l 12a04483b14529bf29873e37fada0d51825d44556261adaf3b5c45a65f9ba935 network default NAD default] ADD starting CNI request [tsf-tas/rekor-redis-7cf5f954b6-5ww4l 12a04483b14529bf29873e37fada0d51825d44556261adaf3b5c45a65f9ba935 network default NAD default] I0330 15:10:04.303293 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-7cf5f954b6-5ww4l, hostIfaceName: 12a04483b14529b, network: default, NAD default, SandboxID: "12a04483b14529bf29873e37fada0d51825d44556261adaf3b5c45a65f9ba935", PCI device ID: , UID: "15fdb8d6-891f-4a1f-89de-42b8ea826946", MAC: 0a:58:0a:81:00:30, IPs: [10.129.0.48/23] I0330 15:10:04.558734 3819 cni.go:424] [tsf-tas/rekor-redis-7cf5f954b6-5ww4l 12a04483b14529bf29873e37fada0d51825d44556261adaf3b5c45a65f9ba935 network default NAD default] ADD finished CNI request [tsf-tas/rekor-redis-7cf5f954b6-5ww4l 12a04483b14529bf29873e37fada0d51825d44556261adaf3b5c45a65f9ba935 network default NAD default], result "{\"interfaces\":[{\"mac\":\"06:b7:ac:5a:07:e3\",\"name\":\"12a04483b14529b\"},{\"mac\":\"0a:58:0a:81:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8657e63e-dec8-49dc-a86a-e1ac971246dc\"}],\"ips\":[{\"address\":\"10.129.0.48/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:10:04.589214 3819 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:04.589899 3819 obj_retry.go:464] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:04.608585 3819 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:04.608623 3819 obj_retry.go:458] Detected object tsf/test-tsf-pipelines of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:04.614072 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-pipelines I0330 15:10:04.614179 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-pipelines I0330 15:10:05.099509 3819 cni.go:403] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm 43ea94a526c403690a366f197e5b4aa2e69d5295646c29834e34665d52b6686e network default NAD default] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm 43ea94a526c403690a366f197e5b4aa2e69d5295646c29834e34665d52b6686e network default NAD default] I0330 15:10:05.195321 3819 cni.go:424] [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm 43ea94a526c403690a366f197e5b4aa2e69d5295646c29834e34665d52b6686e network default NAD default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm 43ea94a526c403690a366f197e5b4aa2e69d5295646c29834e34665d52b6686e network default NAD default], result "{}", err I0330 15:10:05.249494 3819 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:05.249529 3819 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm I0330 15:10:05.251156 3819 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm, ips: 10.129.0.47 I0330 15:10:05.251195 3819 obj_retry.go:464] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:06.019125 3819 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-79895f47c8-wvcmw] creating logical port tsf-tas_rekor-server-79895f47c8-wvcmw for pod on switch ip-10-0-1-27.ec2.internal I0330 15:10:06.019235 3819 kube.go:256] Updating pod tsf-tas/rekor-server-79895f47c8-wvcmw I0330 15:10:06.033373 3819 pod.go:62] [tsf-tas/rekor-server-79895f47c8-wvcmw] pod update took 14.165087ms I0330 15:10:06.033399 3819 base_network_controller_pods.go:937] [default/tsf-tas/rekor-server-79895f47c8-wvcmw] addLogicalPort annotation time took 14.200913ms I0330 15:10:06.034753 3819 pods.go:271] [tsf-tas/rekor-server-79895f47c8-wvcmw] addLogicalPort took 15.644649ms, libovsdb time 939.847µs W0330 15:10:06.103841 3819 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0330 15:10:06.110616 3819 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:06.110647 3819 obj_retry.go:458] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29581390-sghjm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0330 15:10:09.294472 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0330 15:10:11.424871 3819 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:11.425470 3819 obj_retry.go:464] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:11.445958 3819 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:11.445980 3819 obj_retry.go:458] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:11.451529 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0330 15:10:11.451574 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0330 15:10:13.885390 3819 cni.go:403] [tsf-tas/rekor-server-79895f47c8-wvcmw 578ef5a4cbf7530ce5d22c3e9a7403a407cdab66da48a02daa58df51a3bb15ae network default NAD default] ADD starting CNI request [tsf-tas/rekor-server-79895f47c8-wvcmw 578ef5a4cbf7530ce5d22c3e9a7403a407cdab66da48a02daa58df51a3bb15ae network default NAD default] I0330 15:10:13.894023 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-79895f47c8-wvcmw, hostIfaceName: 578ef5a4cbf7530, network: default, NAD default, SandboxID: "578ef5a4cbf7530ce5d22c3e9a7403a407cdab66da48a02daa58df51a3bb15ae", PCI device ID: , UID: "05677000-7643-4342-8182-792e6469a17b", MAC: 0a:58:0a:81:00:31, IPs: [10.129.0.49/23] I0330 15:10:14.152080 3819 cni.go:424] [tsf-tas/rekor-server-79895f47c8-wvcmw 578ef5a4cbf7530ce5d22c3e9a7403a407cdab66da48a02daa58df51a3bb15ae network default NAD default] ADD finished CNI request [tsf-tas/rekor-server-79895f47c8-wvcmw 578ef5a4cbf7530ce5d22c3e9a7403a407cdab66da48a02daa58df51a3bb15ae network default NAD default], result "{\"interfaces\":[{\"mac\":\"96:1c:a6:73:b6:e2\",\"name\":\"578ef5a4cbf7530\"},{\"mac\":\"0a:58:0a:81:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7dca1e6e-b15a-40b9-b75f-49a94594eecd\"}],\"ips\":[{\"address\":\"10.129.0.49/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:10:16.169929 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0330 15:10:17.668094 3819 obj_retry.go:464] Detected object tsf-tpa/create-db-qzhwf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:17.668689 3819 obj_retry.go:464] Detected object tsf-tpa/create-db-qzhwf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0330 15:10:18.151491 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0330 15:10:18.689603 3819 obj_retry.go:458] Detected object tsf-tpa/create-db-qzhwf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:18.689637 3819 obj_retry.go:458] Detected object tsf-tpa/create-db-qzhwf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:21.736609 3819 obj_retry.go:464] Detected object tsf-tpa/migrate-db-cnl7t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:21.737205 3819 obj_retry.go:464] Detected object tsf-tpa/migrate-db-cnl7t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:21.816228 3819 obj_retry.go:458] Detected object tsf-tpa/migrate-db-cnl7t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:21.816260 3819 obj_retry.go:458] Detected object tsf-tpa/migrate-db-cnl7t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:21.987917 3819 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-v7dkr] creating logical port tsf-tpa_create-importers-v7dkr for pod on switch ip-10-0-1-27.ec2.internal I0330 15:10:21.988052 3819 kube.go:256] Updating pod tsf-tpa/create-importers-v7dkr I0330 15:10:22.001895 3819 pod.go:62] [tsf-tpa/create-importers-v7dkr] pod update took 13.866065ms I0330 15:10:22.001928 3819 base_network_controller_pods.go:937] [default/tsf-tpa/create-importers-v7dkr] addLogicalPort annotation time took 13.905189ms I0330 15:10:22.003315 3819 pods.go:271] [tsf-tpa/create-importers-v7dkr] addLogicalPort took 15.418305ms, libovsdb time 934.112µs I0330 15:10:22.352566 3819 cni.go:403] [tsf-tpa/create-importers-v7dkr 0bfc12f21a0e5b0545a340facc04553ff6e7693026a344c6273e2113a41e5cab network default NAD default] ADD starting CNI request [tsf-tpa/create-importers-v7dkr 0bfc12f21a0e5b0545a340facc04553ff6e7693026a344c6273e2113a41e5cab network default NAD default] I0330 15:10:22.360895 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-v7dkr, hostIfaceName: 0bfc12f21a0e5b0, network: default, NAD default, SandboxID: "0bfc12f21a0e5b0545a340facc04553ff6e7693026a344c6273e2113a41e5cab", PCI device ID: , UID: "0620adff-02c9-4c39-afcf-2ed41332bdc6", MAC: 0a:58:0a:81:00:32, IPs: [10.129.0.50/23] I0330 15:10:22.617120 3819 cni.go:424] [tsf-tpa/create-importers-v7dkr 0bfc12f21a0e5b0545a340facc04553ff6e7693026a344c6273e2113a41e5cab network default NAD default] ADD finished CNI request [tsf-tpa/create-importers-v7dkr 0bfc12f21a0e5b0545a340facc04553ff6e7693026a344c6273e2113a41e5cab network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:49:cd:58:00:40\",\"name\":\"0bfc12f21a0e5b0\"},{\"mac\":\"0a:58:0a:81:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1134e88e-148f-4721-9060-7efcac711ac9\"}],\"ips\":[{\"address\":\"10.129.0.50/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:10:27.179332 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0330 15:10:27.240526 3819 cni.go:403] [tsf-tpa/create-importers-v7dkr 0bfc12f21a0e5b0545a340facc04553ff6e7693026a344c6273e2113a41e5cab network default NAD default] DEL starting CNI request [tsf-tpa/create-importers-v7dkr 0bfc12f21a0e5b0545a340facc04553ff6e7693026a344c6273e2113a41e5cab network default NAD default] I0330 15:10:27.352324 3819 cni.go:424] [tsf-tpa/create-importers-v7dkr 0bfc12f21a0e5b0545a340facc04553ff6e7693026a344c6273e2113a41e5cab network default NAD default] DEL finished CNI request [tsf-tpa/create-importers-v7dkr 0bfc12f21a0e5b0545a340facc04553ff6e7693026a344c6273e2113a41e5cab network default NAD default], result "{}", err I0330 15:10:27.405707 3819 obj_retry.go:464] Detected object tsf-tpa/create-importers-v7dkr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:27.405767 3819 pods.go:176] Deleting pod: tsf-tpa/create-importers-v7dkr I0330 15:10:27.407550 3819 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-v7dkr, ips: 10.129.0.50 I0330 15:10:27.407583 3819 obj_retry.go:464] Detected object tsf-tpa/create-importers-v7dkr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:28.427658 3819 obj_retry.go:458] Detected object tsf-tpa/create-importers-v7dkr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:28.427693 3819 obj_retry.go:458] Detected object tsf-tpa/create-importers-v7dkr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0330 15:10:30.605986 3819 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0330 15:10:34.459127 3819 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-rn75p-69g22] creating logical port tsf-tas_tuf-repository-init-rn75p-69g22 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:10:34.459228 3819 kube.go:256] Updating pod tsf-tas/tuf-repository-init-rn75p-69g22 I0330 15:10:34.470335 3819 pod.go:62] [tsf-tas/tuf-repository-init-rn75p-69g22] pod update took 11.131615ms I0330 15:10:34.470362 3819 base_network_controller_pods.go:937] [default/tsf-tas/tuf-repository-init-rn75p-69g22] addLogicalPort annotation time took 11.164602ms I0330 15:10:34.471893 3819 pods.go:271] [tsf-tas/tuf-repository-init-rn75p-69g22] addLogicalPort took 12.780317ms, libovsdb time 1.093939ms I0330 15:10:44.251134 3819 cni.go:403] [tsf-tas/tuf-repository-init-rn75p-69g22 6a24d92460e796f0edacf527af71b3eb198281e93251ceaf3e61d4d1d04e2aad network default NAD default] ADD starting CNI request [tsf-tas/tuf-repository-init-rn75p-69g22 6a24d92460e796f0edacf527af71b3eb198281e93251ceaf3e61d4d1d04e2aad network default NAD default] I0330 15:10:44.259945 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-rn75p-69g22, hostIfaceName: 6a24d92460e796f, network: default, NAD default, SandboxID: "6a24d92460e796f0edacf527af71b3eb198281e93251ceaf3e61d4d1d04e2aad", PCI device ID: , UID: "22fa5369-8533-4922-ab7b-06380eb07c73", MAC: 0a:58:0a:81:00:33, IPs: [10.129.0.51/23] I0330 15:10:44.518011 3819 cni.go:424] [tsf-tas/tuf-repository-init-rn75p-69g22 6a24d92460e796f0edacf527af71b3eb198281e93251ceaf3e61d4d1d04e2aad network default NAD default] ADD finished CNI request [tsf-tas/tuf-repository-init-rn75p-69g22 6a24d92460e796f0edacf527af71b3eb198281e93251ceaf3e61d4d1d04e2aad network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:ed:b8:97:1c:63\",\"name\":\"6a24d92460e796f\"},{\"mac\":\"0a:58:0a:81:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/616467e6-0123-4645-8e8c-169c02213ba6\"}],\"ips\":[{\"address\":\"10.129.0.51/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:10:48.377868 3819 cni.go:403] [tsf-tas/tuf-repository-init-rn75p-69g22 6a24d92460e796f0edacf527af71b3eb198281e93251ceaf3e61d4d1d04e2aad network default NAD default] DEL starting CNI request [tsf-tas/tuf-repository-init-rn75p-69g22 6a24d92460e796f0edacf527af71b3eb198281e93251ceaf3e61d4d1d04e2aad network default NAD default] I0330 15:10:48.486812 3819 cni.go:424] [tsf-tas/tuf-repository-init-rn75p-69g22 6a24d92460e796f0edacf527af71b3eb198281e93251ceaf3e61d4d1d04e2aad network default NAD default] DEL finished CNI request [tsf-tas/tuf-repository-init-rn75p-69g22 6a24d92460e796f0edacf527af71b3eb198281e93251ceaf3e61d4d1d04e2aad network default NAD default], result "{}", err I0330 15:10:48.539347 3819 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-rn75p-69g22 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:48.539382 3819 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-rn75p-69g22 I0330 15:10:48.540947 3819 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-rn75p-69g22, ips: 10.129.0.51 I0330 15:10:48.540980 3819 obj_retry.go:464] Detected object tsf-tas/tuf-repository-init-rn75p-69g22 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:49.560208 3819 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-rn75p-69g22 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:49.560244 3819 obj_retry.go:458] Detected object tsf-tas/tuf-repository-init-rn75p-69g22 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:51.584095 3819 base_network_controller_pods.go:487] [default/tsf-tas/tuf-bc5768c88-jnrhj] creating logical port tsf-tas_tuf-bc5768c88-jnrhj for pod on switch ip-10-0-1-27.ec2.internal I0330 15:10:51.584223 3819 kube.go:256] Updating pod tsf-tas/tuf-bc5768c88-jnrhj I0330 15:10:51.601957 3819 pod.go:62] [tsf-tas/tuf-bc5768c88-jnrhj] pod update took 17.757704ms I0330 15:10:51.601985 3819 base_network_controller_pods.go:937] [default/tsf-tas/tuf-bc5768c88-jnrhj] addLogicalPort annotation time took 17.79157ms I0330 15:10:51.603267 3819 pods.go:271] [tsf-tas/tuf-bc5768c88-jnrhj] addLogicalPort took 19.194742ms, libovsdb time 876.512µs I0330 15:10:54.349763 3819 cni.go:403] [tsf-tas/tuf-bc5768c88-jnrhj d36979d75422df4cc4506a03aeca7a5ab1ccf24b6bf7521f28a38dcac75d1db3 network default NAD default] ADD starting CNI request [tsf-tas/tuf-bc5768c88-jnrhj d36979d75422df4cc4506a03aeca7a5ab1ccf24b6bf7521f28a38dcac75d1db3 network default NAD default] I0330 15:10:54.358619 3819 helper_linux.go:450] ConfigureOVS: namespace: tsf-tas, podName: tuf-bc5768c88-jnrhj, hostIfaceName: d36979d75422df4, network: default, NAD default, SandboxID: "d36979d75422df4cc4506a03aeca7a5ab1ccf24b6bf7521f28a38dcac75d1db3", PCI device ID: , UID: "cc2d430a-5b6a-4a9e-9018-ea64c9e6fe2e", MAC: 0a:58:0a:81:00:34, IPs: [10.129.0.52/23] I0330 15:10:54.615417 3819 cni.go:424] [tsf-tas/tuf-bc5768c88-jnrhj d36979d75422df4cc4506a03aeca7a5ab1ccf24b6bf7521f28a38dcac75d1db3 network default NAD default] ADD finished CNI request [tsf-tas/tuf-bc5768c88-jnrhj d36979d75422df4cc4506a03aeca7a5ab1ccf24b6bf7521f28a38dcac75d1db3 network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:94:5b:f1:37:1f\",\"name\":\"d36979d75422df4\"},{\"mac\":\"0a:58:0a:81:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c74c66dc-f997-4639-b0d1-5b5f7e4bdfe2\"}],\"ips\":[{\"address\":\"10.129.0.52/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:10:56.420766 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0330 15:10:58.673783 3819 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:58.674377 3819 obj_retry.go:464] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:10:58.714621 3819 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:58.714645 3819 obj_retry.go:458] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:10:58.720581 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0330 15:10:58.720609 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0330 15:10:59.403739 3819 namespace.go:98] [integration-service] adding namespace I0330 15:10:59.404691 3819 namespace.go:102] [integration-service] adding namespace took 897.309µs I0330 15:10:59.409988 3819 namespace.go:98] [image-controller] adding namespace I0330 15:10:59.410079 3819 namespace.go:98] [build-service] adding namespace I0330 15:10:59.410789 3819 namespace.go:102] [image-controller] adding namespace took 782.063µs I0330 15:10:59.411345 3819 namespace.go:102] [build-service] adding namespace took 1.246468ms I0330 15:10:59.457568 3819 namespace.go:142] [integration-service] updating namespace I0330 15:10:59.476263 3819 namespace.go:142] [integration-service] updating namespace I0330 15:10:59.484440 3819 namespace.go:142] [image-controller] updating namespace I0330 15:10:59.500860 3819 namespace.go:142] [image-controller] updating namespace I0330 15:10:59.507231 3819 namespace.go:142] [build-service] updating namespace I0330 15:10:59.522184 3819 namespace.go:142] [build-service] updating namespace I0330 15:10:59.691727 3819 namespace.go:142] [build-service] updating namespace I0330 15:10:59.705405 3819 namespace.go:142] [integration-service] updating namespace I0330 15:10:59.771267 3819 namespace.go:98] [release-service] adding namespace I0330 15:10:59.772178 3819 namespace.go:102] [release-service] adding namespace took 886.654µs I0330 15:10:59.809424 3819 namespace.go:142] [konflux-ui] updating namespace I0330 15:10:59.809598 3819 namespace.go:142] [release-service] updating namespace I0330 15:10:59.829812 3819 namespace.go:142] [release-service] updating namespace I0330 15:10:59.934782 3819 namespace.go:98] [konflux-info] adding namespace I0330 15:10:59.935698 3819 namespace.go:102] [konflux-info] adding namespace took 892.626µs I0330 15:10:59.983750 3819 namespace.go:142] [konflux-info] updating namespace I0330 15:11:00.012793 3819 namespace.go:98] [namespace-lister] adding namespace I0330 15:11:00.012844 3819 namespace.go:142] [konflux-info] updating namespace I0330 15:11:00.013697 3819 namespace.go:102] [namespace-lister] adding namespace took 880.135µs I0330 15:11:00.035516 3819 namespace.go:98] [enterprise-contract-service] adding namespace I0330 15:11:00.036275 3819 namespace.go:102] [enterprise-contract-service] adding namespace took 724.775µs I0330 15:11:00.056742 3819 namespace.go:142] [namespace-lister] updating namespace I0330 15:11:00.073067 3819 namespace.go:142] [image-controller] updating namespace I0330 15:11:00.115211 3819 namespace.go:142] [namespace-lister] updating namespace I0330 15:11:00.156500 3819 namespace.go:142] [enterprise-contract-service] updating namespace I0330 15:11:00.188000 3819 namespace.go:142] [enterprise-contract-service] updating namespace I0330 15:11:00.301796 3819 namespace.go:142] [build-service] updating namespace I0330 15:11:00.322007 3819 namespace.go:98] [default-tenant] adding namespace I0330 15:11:00.322769 3819 namespace.go:102] [default-tenant] adding namespace took 743.79µs I0330 15:11:00.399566 3819 namespace.go:98] [segment-bridge] adding namespace I0330 15:11:00.400366 3819 namespace.go:102] [segment-bridge] adding namespace took 771.088µs I0330 15:11:00.419195 3819 namespace.go:142] [default-tenant] updating namespace I0330 15:11:00.449996 3819 namespace.go:142] [default-tenant] updating namespace I0330 15:11:00.535956 3819 namespace.go:142] [segment-bridge] updating namespace I0330 15:11:00.628176 3819 namespace.go:142] [segment-bridge] updating namespace I0330 15:11:00.660316 3819 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-67655c4674-kdpbn] creating logical port namespace-lister_namespace-lister-67655c4674-kdpbn for pod on switch ip-10-0-1-27.ec2.internal I0330 15:11:00.660416 3819 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-kdpbn I0330 15:11:00.704007 3819 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0330 15:11:00.766828 3819 pod.go:62] [namespace-lister/namespace-lister-67655c4674-kdpbn] pod update took 106.435698ms I0330 15:11:00.766857 3819 base_network_controller_pods.go:937] [default/namespace-lister/namespace-lister-67655c4674-kdpbn] addLogicalPort annotation time took 106.473195ms I0330 15:11:00.768208 3819 pods.go:271] [namespace-lister/namespace-lister-67655c4674-kdpbn] addLogicalPort took 107.903903ms, libovsdb time 922.707µs I0330 15:11:00.768313 3819 pod_selector_address_set.go:212] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0330 15:11:00.768328 3819 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0330 15:11:00.771438 3819 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0330 15:11:00.771469 3819 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0330 15:11:00.771482 3819 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0330 15:11:00.772343 3819 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0330 15:11:01.073851 3819 namespace.go:142] [image-controller] updating namespace I0330 15:11:01.437273 3819 base_network_controller_pods.go:487] [default/konflux-ui/dex-7dcf8c4744-cqwhq] creating logical port konflux-ui_dex-7dcf8c4744-cqwhq for pod on switch ip-10-0-1-27.ec2.internal I0330 15:11:01.437391 3819 kube.go:256] Updating pod konflux-ui/dex-7dcf8c4744-cqwhq I0330 15:11:01.704031 3819 pod.go:62] [konflux-ui/dex-7dcf8c4744-cqwhq] pod update took 266.676964ms I0330 15:11:01.704058 3819 base_network_controller_pods.go:937] [default/konflux-ui/dex-7dcf8c4744-cqwhq] addLogicalPort annotation time took 266.710295ms I0330 15:11:01.705313 3819 pods.go:271] [konflux-ui/dex-7dcf8c4744-cqwhq] addLogicalPort took 268.060886ms, libovsdb time 832.878µs I0330 15:11:01.993834 3819 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l] creating logical port image-controller_image-controller-controller-manager-5f8cd64d55-8dl5l for pod on switch ip-10-0-1-27.ec2.internal I0330 15:11:01.993933 3819 kube.go:256] Updating pod image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l I0330 15:11:02.098038 3819 pod.go:62] [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l] pod update took 104.126549ms I0330 15:11:02.098069 3819 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l] addLogicalPort annotation time took 104.166039ms I0330 15:11:02.099625 3819 pods.go:271] [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l] addLogicalPort took 105.808035ms, libovsdb time 1.117809ms I0330 15:11:02.107614 3819 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-759977d69f-d2h5z] creating logical port integration-service_integration-service-controller-manager-759977d69f-d2h5z for pod on switch ip-10-0-1-27.ec2.internal I0330 15:11:02.107692 3819 kube.go:256] Updating pod integration-service/integration-service-controller-manager-759977d69f-d2h5z I0330 15:11:02.142982 3819 pod.go:62] [integration-service/integration-service-controller-manager-759977d69f-d2h5z] pod update took 35.306841ms I0330 15:11:02.143009 3819 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-759977d69f-d2h5z] addLogicalPort annotation time took 35.340282ms I0330 15:11:02.144333 3819 pods.go:271] [integration-service/integration-service-controller-manager-759977d69f-d2h5z] addLogicalPort took 36.72999ms, libovsdb time 863.998µs I0330 15:11:02.242199 3819 namespace.go:142] [integration-service] updating namespace I0330 15:11:02.473961 3819 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7df75dd4df-j5p77] creating logical port build-service_build-service-controller-manager-7df75dd4df-j5p77 for pod on switch ip-10-0-1-27.ec2.internal I0330 15:11:02.474061 3819 kube.go:256] Updating pod build-service/build-service-controller-manager-7df75dd4df-j5p77 I0330 15:11:02.498003 3819 pod.go:62] [build-service/build-service-controller-manager-7df75dd4df-j5p77] pod update took 23.967544ms I0330 15:11:02.498028 3819 base_network_controller_pods.go:937] [default/build-service/build-service-controller-manager-7df75dd4df-j5p77] addLogicalPort annotation time took 23.998483ms I0330 15:11:02.499336 3819 pods.go:271] [build-service/build-service-controller-manager-7df75dd4df-j5p77] addLogicalPort took 25.392087ms, libovsdb time 891.891µs I0330 15:11:02.529787 3819 namespace.go:142] [release-service] updating namespace I0330 15:11:02.614112 3819 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l 48b24a6cc9025fa8f14de4edb04db0fab527e847f9d73dfbce3b04d15e27e229 network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l 48b24a6cc9025fa8f14de4edb04db0fab527e847f9d73dfbce3b04d15e27e229 network default NAD default] I0330 15:11:02.622991 3819 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-5f8cd64d55-8dl5l, hostIfaceName: 48b24a6cc9025fa, network: default, NAD default, SandboxID: "48b24a6cc9025fa8f14de4edb04db0fab527e847f9d73dfbce3b04d15e27e229", PCI device ID: , UID: "2d3f5737-2633-42b4-960d-1217e4f20f27", MAC: 0a:58:0a:81:00:37, IPs: [10.129.0.55/23] I0330 15:11:02.665851 3819 namespace.go:142] [build-service] updating namespace I0330 15:11:02.678436 3819 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l 48b24a6cc9025fa8f14de4edb04db0fab527e847f9d73dfbce3b04d15e27e229 network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l 48b24a6cc9025fa8f14de4edb04db0fab527e847f9d73dfbce3b04d15e27e229 network default NAD default], result "{\"interfaces\":[{\"mac\":\"b2:ca:b6:27:b3:2c\",\"name\":\"48b24a6cc9025fa\"},{\"mac\":\"0a:58:0a:81:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c0a9c622-2b6d-4d12-8082-ab582b91c0ea\"}],\"ips\":[{\"address\":\"10.129.0.55/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:11:02.692583 3819 namespace.go:142] [image-controller] updating namespace I0330 15:11:02.708210 3819 namespace.go:142] [integration-service] updating namespace I0330 15:11:02.721776 3819 namespace.go:142] [release-service] updating namespace I0330 15:11:02.732550 3819 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-854f775895-kpnbp] creating logical port release-service_release-service-controller-manager-854f775895-kpnbp for pod on switch ip-10-0-1-27.ec2.internal I0330 15:11:02.732663 3819 kube.go:256] Updating pod release-service/release-service-controller-manager-854f775895-kpnbp I0330 15:11:02.749513 3819 pod.go:62] [release-service/release-service-controller-manager-854f775895-kpnbp] pod update took 16.88743ms I0330 15:11:02.749554 3819 base_network_controller_pods.go:937] [default/release-service/release-service-controller-manager-854f775895-kpnbp] addLogicalPort annotation time took 16.934558ms I0330 15:11:02.750937 3819 pods.go:271] [release-service/release-service-controller-manager-854f775895-kpnbp] addLogicalPort took 18.416069ms, libovsdb time 876.279µs I0330 15:11:02.805064 3819 namespace.go:142] [build-service] updating namespace I0330 15:11:02.810383 3819 base_network_controller_pods.go:487] [default/konflux-ui/proxy-657c87b487-hfqfj] creating logical port konflux-ui_proxy-657c87b487-hfqfj for pod on switch ip-10-0-1-27.ec2.internal I0330 15:11:02.810521 3819 kube.go:256] Updating pod konflux-ui/proxy-657c87b487-hfqfj I0330 15:11:02.834793 3819 pod.go:62] [konflux-ui/proxy-657c87b487-hfqfj] pod update took 24.30594ms I0330 15:11:02.834818 3819 base_network_controller_pods.go:937] [default/konflux-ui/proxy-657c87b487-hfqfj] addLogicalPort annotation time took 24.340375ms I0330 15:11:02.836323 3819 pods.go:271] [konflux-ui/proxy-657c87b487-hfqfj] addLogicalPort took 25.965008ms, libovsdb time 1.061749ms I0330 15:11:02.847295 3819 cni.go:403] [build-service/build-service-controller-manager-7df75dd4df-j5p77 357cbf9615a66b7ca0055230baf3e3e810e399d742c8e769cafd1fe576ec4a5c network default NAD default] ADD starting CNI request [build-service/build-service-controller-manager-7df75dd4df-j5p77 357cbf9615a66b7ca0055230baf3e3e810e399d742c8e769cafd1fe576ec4a5c network default NAD default] I0330 15:11:02.856927 3819 helper_linux.go:450] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7df75dd4df-j5p77, hostIfaceName: 357cbf9615a66b7, network: default, NAD default, SandboxID: "357cbf9615a66b7ca0055230baf3e3e810e399d742c8e769cafd1fe576ec4a5c", PCI device ID: , UID: "36095f76-a029-4d53-9079-09d9228f664b", MAC: 0a:58:0a:81:00:39, IPs: [10.129.0.57/23] I0330 15:11:02.907327 3819 cni.go:424] [build-service/build-service-controller-manager-7df75dd4df-j5p77 357cbf9615a66b7ca0055230baf3e3e810e399d742c8e769cafd1fe576ec4a5c network default NAD default] ADD finished CNI request [build-service/build-service-controller-manager-7df75dd4df-j5p77 357cbf9615a66b7ca0055230baf3e3e810e399d742c8e769cafd1fe576ec4a5c network default NAD default], result "{\"interfaces\":[{\"mac\":\"7e:d8:55:22:77:b7\",\"name\":\"357cbf9615a66b7\"},{\"mac\":\"0a:58:0a:81:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3a067cf3-25eb-40ec-93d8-048214ef3d06\"}],\"ips\":[{\"address\":\"10.129.0.57/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:11:02.917847 3819 namespace.go:142] [image-controller] updating namespace I0330 15:11:02.933057 3819 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-cfntm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:11:02.933662 3819 obj_retry.go:464] Detected object konflux-ui/tsf-konflux-public-key-cfntm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:11:02.949553 3819 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-5969f69546-wrqvp] creating logical port integration-service_integration-service-controller-manager-5969f69546-wrqvp for pod on switch ip-10-0-1-27.ec2.internal I0330 15:11:02.949660 3819 kube.go:256] Updating pod integration-service/integration-service-controller-manager-5969f69546-wrqvp I0330 15:11:02.965271 3819 pod.go:62] [integration-service/integration-service-controller-manager-5969f69546-wrqvp] pod update took 15.639099ms I0330 15:11:02.965302 3819 base_network_controller_pods.go:937] [default/integration-service/integration-service-controller-manager-5969f69546-wrqvp] addLogicalPort annotation time took 15.678374ms I0330 15:11:02.966961 3819 pods.go:271] [integration-service/integration-service-controller-manager-5969f69546-wrqvp] addLogicalPort took 17.453695ms, libovsdb time 1.046742ms I0330 15:11:03.008219 3819 namespace.go:142] [integration-service] updating namespace I0330 15:11:03.065176 3819 namespace.go:142] [release-service] updating namespace I0330 15:11:03.130895 3819 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-cfntm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:11:03.130946 3819 obj_retry.go:458] Detected object konflux-ui/tsf-konflux-public-key-cfntm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:11:04.300119 3819 namespace.go:142] [default-tenant] updating namespace I0330 15:11:04.501519 3819 namespace.go:142] [enterprise-contract-service] updating namespace I0330 15:11:04.699222 3819 namespace.go:142] [konflux-info] updating namespace I0330 15:11:04.903267 3819 namespace.go:142] [namespace-lister] updating namespace I0330 15:11:05.101696 3819 namespace.go:142] [segment-bridge] updating namespace I0330 15:11:05.223037 3819 namespace.go:142] [default-tenant] updating namespace I0330 15:11:05.239485 3819 namespace.go:142] [enterprise-contract-service] updating namespace I0330 15:11:05.253830 3819 namespace.go:142] [konflux-info] updating namespace I0330 15:11:05.269960 3819 namespace.go:142] [namespace-lister] updating namespace I0330 15:11:05.282972 3819 namespace.go:142] [segment-bridge] updating namespace I0330 15:11:05.326436 3819 namespace.go:142] [default-tenant] updating namespace I0330 15:11:05.414283 3819 namespace.go:142] [enterprise-contract-service] updating namespace I0330 15:11:05.510841 3819 namespace.go:142] [konflux-info] updating namespace I0330 15:11:05.607196 3819 namespace.go:142] [namespace-lister] updating namespace I0330 15:11:05.710152 3819 namespace.go:142] [segment-bridge] updating namespace I0330 15:11:06.359824 3819 cni.go:403] [namespace-lister/namespace-lister-67655c4674-kdpbn f475e3942631646849366a5029de7f22cf2bfe7ad63083c0c63f9ccb7712f897 network default NAD default] ADD starting CNI request [namespace-lister/namespace-lister-67655c4674-kdpbn f475e3942631646849366a5029de7f22cf2bfe7ad63083c0c63f9ccb7712f897 network default NAD default] I0330 15:11:06.368983 3819 helper_linux.go:450] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-67655c4674-kdpbn, hostIfaceName: f475e3942631646, network: default, NAD default, SandboxID: "f475e3942631646849366a5029de7f22cf2bfe7ad63083c0c63f9ccb7712f897", PCI device ID: , UID: "d3ccc06d-23cc-41fb-a8a9-4da6538deeac", MAC: 0a:58:0a:81:00:35, IPs: [10.129.0.53/23] I0330 15:11:06.425684 3819 cni.go:424] [namespace-lister/namespace-lister-67655c4674-kdpbn f475e3942631646849366a5029de7f22cf2bfe7ad63083c0c63f9ccb7712f897 network default NAD default] ADD finished CNI request [namespace-lister/namespace-lister-67655c4674-kdpbn f475e3942631646849366a5029de7f22cf2bfe7ad63083c0c63f9ccb7712f897 network default NAD default], result "{\"interfaces\":[{\"mac\":\"9a:da:5a:a1:04:c3\",\"name\":\"f475e3942631646\"},{\"mac\":\"0a:58:0a:81:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d2816598-9828-4f0d-a0ae-94b87d44eb39\"}],\"ips\":[{\"address\":\"10.129.0.53/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:11:07.431675 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf W0330 15:11:07.495125 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0330 15:11:07.521812 3819 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0330 15:11:09.290429 3819 cni.go:403] [konflux-ui/dex-7dcf8c4744-cqwhq c46dd831b3e75a7b7cb0ff6c9499b932eb0fd037ab705fc2b47fb9d52aa8e3a5 network default NAD default] ADD starting CNI request [konflux-ui/dex-7dcf8c4744-cqwhq c46dd831b3e75a7b7cb0ff6c9499b932eb0fd037ab705fc2b47fb9d52aa8e3a5 network default NAD default] I0330 15:11:09.299113 3819 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: dex-7dcf8c4744-cqwhq, hostIfaceName: c46dd831b3e75a7, network: default, NAD default, SandboxID: "c46dd831b3e75a7b7cb0ff6c9499b932eb0fd037ab705fc2b47fb9d52aa8e3a5", PCI device ID: , UID: "2f0313a0-2c59-46a2-a929-af8af954f35d", MAC: 0a:58:0a:81:00:36, IPs: [10.129.0.54/23] I0330 15:11:09.351303 3819 cni.go:424] [konflux-ui/dex-7dcf8c4744-cqwhq c46dd831b3e75a7b7cb0ff6c9499b932eb0fd037ab705fc2b47fb9d52aa8e3a5 network default NAD default] ADD finished CNI request [konflux-ui/dex-7dcf8c4744-cqwhq c46dd831b3e75a7b7cb0ff6c9499b932eb0fd037ab705fc2b47fb9d52aa8e3a5 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:91:d0:f9:95:11\",\"name\":\"c46dd831b3e75a7\"},{\"mac\":\"0a:58:0a:81:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e398151a-4e32-4f9c-996c-895200b887dd\"}],\"ips\":[{\"address\":\"10.129.0.54/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:11:09.510456 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0330 15:11:09.534665 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0330 15:11:10.850743 3819 cni.go:403] [konflux-ui/proxy-657c87b487-hfqfj 999e9f62f425e6b6d1b58692528cd7ce8c41514f43d33aa46733bd407ead576b network default NAD default] ADD starting CNI request [konflux-ui/proxy-657c87b487-hfqfj 999e9f62f425e6b6d1b58692528cd7ce8c41514f43d33aa46733bd407ead576b network default NAD default] I0330 15:11:10.851519 3819 cni.go:403] [release-service/release-service-controller-manager-854f775895-kpnbp 2256229919d55b3124563a8040db4fdc48a6e2425c9424ca0680bdd5b2051ea0 network default NAD default] ADD starting CNI request [release-service/release-service-controller-manager-854f775895-kpnbp 2256229919d55b3124563a8040db4fdc48a6e2425c9424ca0680bdd5b2051ea0 network default NAD default] I0330 15:11:10.861367 3819 helper_linux.go:450] ConfigureOVS: namespace: konflux-ui, podName: proxy-657c87b487-hfqfj, hostIfaceName: 999e9f62f425e6b, network: default, NAD default, SandboxID: "999e9f62f425e6b6d1b58692528cd7ce8c41514f43d33aa46733bd407ead576b", PCI device ID: , UID: "a3bec3f7-4eb5-4614-8305-2fd32f9496ba", MAC: 0a:58:0a:81:00:3b, IPs: [10.129.0.59/23] I0330 15:11:10.861686 3819 helper_linux.go:450] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-854f775895-kpnbp, hostIfaceName: 2256229919d55b3, network: default, NAD default, SandboxID: "2256229919d55b3124563a8040db4fdc48a6e2425c9424ca0680bdd5b2051ea0", PCI device ID: , UID: "adb4a3f9-bf70-490e-b6f5-9d2cc60c9eaf", MAC: 0a:58:0a:81:00:3a, IPs: [10.129.0.58/23] I0330 15:11:10.909911 3819 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-d2h5z 2de73c618ef97aa1075ec95a1b585b55c3e0b28b925bfd0185f8ffc974189900 network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-759977d69f-d2h5z 2de73c618ef97aa1075ec95a1b585b55c3e0b28b925bfd0185f8ffc974189900 network default NAD default] I0330 15:11:10.918116 3819 cni.go:424] [konflux-ui/proxy-657c87b487-hfqfj 999e9f62f425e6b6d1b58692528cd7ce8c41514f43d33aa46733bd407ead576b network default NAD default] ADD finished CNI request [konflux-ui/proxy-657c87b487-hfqfj 999e9f62f425e6b6d1b58692528cd7ce8c41514f43d33aa46733bd407ead576b network default NAD default], result "{\"interfaces\":[{\"mac\":\"46:b5:f5:a2:d1:18\",\"name\":\"999e9f62f425e6b\"},{\"mac\":\"0a:58:0a:81:00:3b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/41e4b525-77d0-4d61-8c38-1921537406b6\"}],\"ips\":[{\"address\":\"10.129.0.59/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:11:10.921064 3819 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-759977d69f-d2h5z, hostIfaceName: 2de73c618ef97aa, network: default, NAD default, SandboxID: "2de73c618ef97aa1075ec95a1b585b55c3e0b28b925bfd0185f8ffc974189900", PCI device ID: , UID: "aaf024e5-f2f8-472c-a222-eb2c8dbc841a", MAC: 0a:58:0a:81:00:38, IPs: [10.129.0.56/23] I0330 15:11:11.118720 3819 cni.go:403] [integration-service/integration-service-controller-manager-5969f69546-wrqvp 79437808a79bbfc71c17522057d05ed101148a6b98b9ed00fea046f157d7bbc1 network default NAD default] ADD starting CNI request [integration-service/integration-service-controller-manager-5969f69546-wrqvp 79437808a79bbfc71c17522057d05ed101148a6b98b9ed00fea046f157d7bbc1 network default NAD default] I0330 15:11:11.127746 3819 helper_linux.go:450] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-5969f69546-wrqvp, hostIfaceName: 79437808a79bbfc, network: default, NAD default, SandboxID: "79437808a79bbfc71c17522057d05ed101148a6b98b9ed00fea046f157d7bbc1", PCI device ID: , UID: "e3de1c7b-c191-4cde-a59c-2400a8462755", MAC: 0a:58:0a:81:00:3c, IPs: [10.129.0.60/23] I0330 15:11:11.148555 3819 cni.go:424] [release-service/release-service-controller-manager-854f775895-kpnbp 2256229919d55b3124563a8040db4fdc48a6e2425c9424ca0680bdd5b2051ea0 network default NAD default] ADD finished CNI request [release-service/release-service-controller-manager-854f775895-kpnbp 2256229919d55b3124563a8040db4fdc48a6e2425c9424ca0680bdd5b2051ea0 network default NAD default], result "{\"interfaces\":[{\"mac\":\"16:8b:1f:62:4a:a3\",\"name\":\"2256229919d55b3\"},{\"mac\":\"0a:58:0a:81:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4a33e766-b77c-4fcb-a1f7-10678be89741\"}],\"ips\":[{\"address\":\"10.129.0.58/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:11:11.190623 3819 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-d2h5z 2de73c618ef97aa1075ec95a1b585b55c3e0b28b925bfd0185f8ffc974189900 network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-759977d69f-d2h5z 2de73c618ef97aa1075ec95a1b585b55c3e0b28b925bfd0185f8ffc974189900 network default NAD default], result "{\"interfaces\":[{\"mac\":\"8a:24:d8:48:83:98\",\"name\":\"2de73c618ef97aa\"},{\"mac\":\"0a:58:0a:81:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/20c1df3d-2a31-40a6-a87b-15b19522d725\"}],\"ips\":[{\"address\":\"10.129.0.56/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:11:11.388006 3819 cni.go:424] [integration-service/integration-service-controller-manager-5969f69546-wrqvp 79437808a79bbfc71c17522057d05ed101148a6b98b9ed00fea046f157d7bbc1 network default NAD default] ADD finished CNI request [integration-service/integration-service-controller-manager-5969f69546-wrqvp 79437808a79bbfc71c17522057d05ed101148a6b98b9ed00fea046f157d7bbc1 network default NAD default], result "{\"interfaces\":[{\"mac\":\"3a:2e:61:4c:a0:9f\",\"name\":\"79437808a79bbfc\"},{\"mac\":\"0a:58:0a:81:00:3c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/206f0249-12ce-48bf-afd6-0fdfc582e279\"}],\"ips\":[{\"address\":\"10.129.0.60/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0330 15:11:11.530925 3819 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0330 15:11:15.668993 3819 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0330 15:11:15.669018 3819 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0330 15:11:18.504875 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0330 15:11:18.530895 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0330 15:11:18.596289 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0330 15:11:18.600134 3819 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0330 15:11:18.626134 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0330 15:11:18.626410 3819 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0330 15:11:18.681239 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0330 15:11:18.683689 3819 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0330 15:11:18.717604 3819 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0330 15:11:18.717624 3819 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy I0330 15:11:26.142354 3819 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 W0330 15:11:29.611497 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0330 15:11:29.642472 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0330 15:11:29.642670 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0330 15:11:29.667811 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0330 15:11:29.670752 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service I0330 15:11:29.894943 3819 cni.go:403] [integration-service/integration-service-controller-manager-759977d69f-d2h5z 2de73c618ef97aa1075ec95a1b585b55c3e0b28b925bfd0185f8ffc974189900 network default NAD default] DEL starting CNI request [integration-service/integration-service-controller-manager-759977d69f-d2h5z 2de73c618ef97aa1075ec95a1b585b55c3e0b28b925bfd0185f8ffc974189900 network default NAD default] I0330 15:11:29.990735 3819 cni.go:424] [integration-service/integration-service-controller-manager-759977d69f-d2h5z 2de73c618ef97aa1075ec95a1b585b55c3e0b28b925bfd0185f8ffc974189900 network default NAD default] DEL finished CNI request [integration-service/integration-service-controller-manager-759977d69f-d2h5z 2de73c618ef97aa1075ec95a1b585b55c3e0b28b925bfd0185f8ffc974189900 network default NAD default], result "{}", err I0330 15:11:30.049733 3819 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-d2h5z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:11:30.049776 3819 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-759977d69f-d2h5z I0330 15:11:30.051230 3819 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-759977d69f-d2h5z, ips: 10.129.0.56 I0330 15:11:30.051265 3819 obj_retry.go:464] Detected object integration-service/integration-service-controller-manager-759977d69f-d2h5z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:11:30.687037 3819 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-d2h5z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:11:30.687069 3819 obj_retry.go:458] Detected object integration-service/integration-service-controller-manager-759977d69f-d2h5z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:11:30.690489 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-759977d69f-d2h5z I0330 15:11:30.690515 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-759977d69f-d2h5z W0330 15:11:53.730411 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0330 15:11:53.730439 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0330 15:11:53.730452 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0330 15:11:53.730457 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0330 15:11:55.677149 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0330 15:11:55.677179 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0330 15:12:07.947496 3819 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0330 15:12:07.947516 3819 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0330 15:12:07.947519 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0330 15:12:07.947594 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0330 15:12:07.947630 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0330 15:12:07.947654 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0330 15:12:19.958576 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0330 15:12:19.961521 3819 util.go:959] Failed to build node endpoints for node ip-10-0-1-27.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0330 15:12:25.331046 3819 obj_retry.go:464] Detected object konflux-ui/proxy-688fb895fd-pztfd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:12:25.331686 3819 obj_retry.go:464] Detected object konflux-ui/proxy-688fb895fd-pztfd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:12:26.050633 3819 obj_retry.go:458] Detected object konflux-ui/proxy-688fb895fd-pztfd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:12:26.050683 3819 obj_retry.go:458] Detected object konflux-ui/proxy-688fb895fd-pztfd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:12:26.055725 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-688fb895fd-pztfd I0330 15:12:26.055752 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-688fb895fd-pztfd I0330 15:12:32.223405 3819 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:12:32.224055 3819 obj_retry.go:464] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:12:32.245423 3819 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:12:32.245449 3819 obj_retry.go:458] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:12:32.251592 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0330 15:12:32.251619 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0330 15:13:01.838825 3819 cni.go:403] [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk 04fd4f26269a2482836efee8f54f168f401f34f78f8fbb697408952c99fc62ce network default NAD default] DEL starting CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk 04fd4f26269a2482836efee8f54f168f401f34f78f8fbb697408952c99fc62ce network default NAD default] I0330 15:13:01.939904 3819 cni.go:424] [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk 04fd4f26269a2482836efee8f54f168f401f34f78f8fbb697408952c99fc62ce network default NAD default] DEL finished CNI request [konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk 04fd4f26269a2482836efee8f54f168f401f34f78f8fbb697408952c99fc62ce network default NAD default], result "{}", err I0330 15:13:02.007280 3819 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:13:02.007318 3819 pods.go:176] Deleting pod: konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk I0330 15:13:02.008871 3819 pods.go:236] Attempting to release IPs for pod: konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk, ips: 10.129.0.33 I0330 15:13:02.008902 3819 obj_retry.go:464] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:13:02.052760 3819 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-56b684c87-k9gbl] creating logical port image-controller_image-controller-controller-manager-56b684c87-k9gbl for pod on switch ip-10-0-1-27.ec2.internal I0330 15:13:02.052894 3819 kube.go:256] Updating pod image-controller/image-controller-controller-manager-56b684c87-k9gbl I0330 15:13:02.069427 3819 pod.go:62] [image-controller/image-controller-controller-manager-56b684c87-k9gbl] pod update took 16.561294ms I0330 15:13:02.069456 3819 base_network_controller_pods.go:937] [default/image-controller/image-controller-controller-manager-56b684c87-k9gbl] addLogicalPort annotation time took 16.594967ms I0330 15:13:02.070996 3819 pods.go:271] [image-controller/image-controller-controller-manager-56b684c87-k9gbl] addLogicalPort took 18.262277ms, libovsdb time 1.034602ms I0330 15:13:02.289415 3819 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:13:02.289441 3819 obj_retry.go:458] Detected object konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:13:02.295983 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk I0330 15:13:02.296019 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod konflux-operator/konflux-operator-controller-manager-7db75c44-92hpk I0330 15:13:02.419260 3819 cni.go:403] [image-controller/image-controller-controller-manager-56b684c87-k9gbl 480b26ba45826f8734ab2626782d15801948ba3c8aee18ef345fde64cb38ad21 network default NAD default] ADD starting CNI request [image-controller/image-controller-controller-manager-56b684c87-k9gbl 480b26ba45826f8734ab2626782d15801948ba3c8aee18ef345fde64cb38ad21 network default NAD default] I0330 15:13:02.428582 3819 helper_linux.go:450] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-56b684c87-k9gbl, hostIfaceName: 480b26ba45826f8, network: default, NAD default, SandboxID: "480b26ba45826f8734ab2626782d15801948ba3c8aee18ef345fde64cb38ad21", PCI device ID: , UID: "c47d0637-0ff5-4f22-ad2a-5628a987d48b", MAC: 0a:58:0a:81:00:3d, IPs: [10.129.0.61/23] I0330 15:13:02.480923 3819 cni.go:424] [image-controller/image-controller-controller-manager-56b684c87-k9gbl 480b26ba45826f8734ab2626782d15801948ba3c8aee18ef345fde64cb38ad21 network default NAD default] ADD finished CNI request [image-controller/image-controller-controller-manager-56b684c87-k9gbl 480b26ba45826f8734ab2626782d15801948ba3c8aee18ef345fde64cb38ad21 network default NAD default], result "{\"interfaces\":[{\"mac\":\"a2:ce:cf:34:53:da\",\"name\":\"480b26ba45826f8\"},{\"mac\":\"0a:58:0a:81:00:3d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/70931a55-9332-4195-9889-d8f6cf5f09b6\"}],\"ips\":[{\"address\":\"10.129.0.61/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0330 15:13:15.509224 3819 cni.go:403] [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l 48b24a6cc9025fa8f14de4edb04db0fab527e847f9d73dfbce3b04d15e27e229 network default NAD default] DEL starting CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l 48b24a6cc9025fa8f14de4edb04db0fab527e847f9d73dfbce3b04d15e27e229 network default NAD default] I0330 15:13:15.613883 3819 cni.go:424] [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l 48b24a6cc9025fa8f14de4edb04db0fab527e847f9d73dfbce3b04d15e27e229 network default NAD default] DEL finished CNI request [image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l 48b24a6cc9025fa8f14de4edb04db0fab527e847f9d73dfbce3b04d15e27e229 network default NAD default], result "{}", err I0330 15:13:15.675760 3819 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0330 15:13:15.675794 3819 pods.go:176] Deleting pod: image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l I0330 15:13:15.677486 3819 pods.go:236] Attempting to release IPs for pod: image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l, ips: 10.129.0.55 I0330 15:13:15.677529 3819 obj_retry.go:464] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0330 15:13:16.131313 3819 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0330 15:13:16.382150 3819 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:13:16.382177 3819 obj_retry.go:458] Detected object image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0330 15:13:16.387889 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *factory.egressIPPod image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l I0330 15:13:16.387920 3819 obj_retry.go:737] Ignoring delete event for resource in terminal state *v1.Pod image-controller/image-controller-controller-manager-5f8cd64d55-8dl5l I0330 15:13:28.160014 3819 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0330 15:13:38.044452 3819 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=59 I0330 15:13:40.166765 3819 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10