+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-7.ec2.internal ++ [[ -n ip-10-0-2-7.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-7.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0428 15:37:21.920110544 - disable conntrack on geneve port' I0428 15:37:21.920110544 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0428 15:37:21.937592469 - starting ovnkube-node' I0428 15:37:21.937592469 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-7.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-7.ec2.internal --init-node ip-10-0-2-7.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-7.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0428 15:37:21.978769 3786 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0428 15:37:21.978829 3786 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-6952a9a844.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0428 15:37:21.980277 3786 kube.go:431] Waiting for certificate I0428 15:37:21.980313 3786 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0428 15:37:21.980347 3786 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0428 15:37:21.981081 3786 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 15:37:22.011460 3786 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:22.011479 3786 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:22.014881 3786 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:22.018193 3786 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-r2ltm" I0428 15:37:22.022468 3786 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-r2ltm" I0428 15:37:22.022555 3786 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:22.980834 3786 kube.go:438] Certificate found I0428 15:37:22.981782 3786 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 15:37:22.983457 3786 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0428 15:37:23.031973 3786 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-29 15:32:22 +0000 UTC" deadline="2026-04-29 12:14:46.107921201 +0000 UTC" I0428 15:37:23.032024 3786 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h37m23.075913384s" I0428 15:37:23.055693 3786 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 15:37:23.075808 3786 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0428 15:37:23.075843 3786 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0428 15:37:23.075903 3786 server.go:101] MetricServer registers OVS metrics I0428 15:37:23.076061 3786 factory.go:561] Starting watch factory I0428 15:37:23.076199 3786 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076225 3786 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076288 3786 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076312 3786 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076440 3786 server.go:105] MetricServer registers OVN DB metrics I0428 15:37:23.076477 3786 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076521 3786 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076558 3786 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076566 3786 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076593 3786 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076595 3786 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076601 3786 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.076614 3786 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.077322 3786 controller_manager.go:370] Starting the ovnkube controller I0428 15:37:23.077335 3786 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-7.ec2.internal I0428 15:37:23.077398 3786 controller_manager.go:395] NBDB zone sync took: 53.451µs I0428 15:37:23.077410 3786 factory.go:561] Starting watch factory I0428 15:37:23.081403 3786 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0428 15:37:23.081428 3786 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0428 15:37:23.093667 3786 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.095013 3786 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0428 15:37:23.095037 3786 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0428 15:37:23.095807 3786 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.097213 3786 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.098141 3786 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.098505 3786 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.100241 3786 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0428 15:37:23.100341 3786 server.go:109] MetricServer registers OVN Controller metrics I0428 15:37:23.105368 3786 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:23.106323 3786 server.go:113] MetricServer registers OVN Northd metrics I0428 15:37:23.111292 3786 metrics.go:519] OVN Metrics Server starts to run ... I0428 15:37:23.182051 3786 factory.go:1977] *v1.Pod informer cache synced successfully I0428 15:37:23.182079 3786 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 15:37:23.182084 3786 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 15:37:23.182106 3786 factory.go:1977] *v1.Node informer cache synced successfully I0428 15:37:23.182110 3786 factory.go:1977] *v1.Service informer cache synced successfully I0428 15:37:23.182116 3786 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 15:37:23.182223 3786 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:23.182268 3786 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:23.182274 3786 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:23.182308 3786 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:23.182550 3786 factory.go:1977] *v1.Pod informer cache synced successfully I0428 15:37:23.182562 3786 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 15:37:23.182567 3786 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 15:37:23.182570 3786 factory.go:1977] *v1.Node informer cache synced successfully I0428 15:37:23.182574 3786 factory.go:1977] *v1.Service informer cache synced successfully I0428 15:37:23.182577 3786 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 15:37:23.184900 3786 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:23.185206 3786 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:23.187251 3786 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 15:37:23.187262 3786 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 15:37:23.187320 3786 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.187344 3786 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.187827 3786 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 15:37:23.187841 3786 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 15:37:23.189516 3786 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.192846 3786 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 15:37:23.192898 3786 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.192908 3786 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.192974 3786 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 15:37:23.194976 3786 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.198035 3786 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 15:37:23.198050 3786 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 15:37:23.198128 3786 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.198144 3786 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.200422 3786 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.203508 3786 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 15:37:23.203566 3786 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.203576 3786 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.203637 3786 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 15:37:23.205662 3786 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.208741 3786 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 15:37:23.208751 3786 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 15:37:23.208800 3786 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.208812 3786 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.211022 3786 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.213962 3786 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 15:37:23.213970 3786 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 15:37:23.214221 3786 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:37:23.214234 3786 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:37:23.216714 3786 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:37:23.219945 3786 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 15:37:23.219953 3786 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 15:37:23.220030 3786 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.220050 3786 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.220074 3786 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.220137 3786 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.222377 3786 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.222624 3786 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:23.225938 3786 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 15:37:23.225948 3786 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 15:37:23.225956 3786 factory.go:680] Watch Factory start up complete, took: 148.545597ms I0428 15:37:23.225963 3786 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-7.ec2.internal" zone I0428 15:37:23.226017 3786 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 15:37:23.226025 3786 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 15:37:23.226031 3786 factory.go:680] Watch Factory start up complete, took: 149.979107ms I0428 15:37:23.226103 3786 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0428 15:37:23.275246 3786 default_node_network_controller.go:710] Initializing the default node network controller I0428 15:37:23.351614 3786 udn_isolation.go:101] Starting UDN host isolation manager I0428 15:37:23.357698 3786 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0428 15:37:23.426910 3786 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0428 15:37:23.427021 3786 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0428 15:37:23.427047 3786 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0428 15:37:23.458973 3786 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0428 15:37:23.459041 3786 default_node_network_controller.go:809] Node ip-10-0-2-7.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0428 15:37:23.464226 3786 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.7"] k8s.ovn.org/zone-name:ip-10-0-2-7.ec2.internal] on node ip-10-0-2-7.ec2.internal I0428 15:37:23.486116 3786 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0428 15:37:23.492393 3786 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0428 15:37:23.492500 3786 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0428 15:37:23.492627 3786 gateway_init.go:263] Preparing Gateway I0428 15:37:23.492639 3786 gateway_shared_intf.go:1687] Creating new gateway I0428 15:37:23.611739 3786 iptables.go:108] Creating table: filter chain: FORWARD I0428 15:37:23.618758 3786 iptables.go:108] Creating table: filter chain: OUTPUT I0428 15:37:23.625451 3786 gateway_shared_intf.go:1816] Gateway Creation Complete I0428 15:37:23.625468 3786 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-7.ec2.internal","mac-address":"0e:99:b9:48:ab:0b","ip-addresses":["10.0.2.7/24"],"ip-address":"10.0.2.7/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:2947311f-9ee9-4ee8-800d-ae9d29b50ea8 k8s.ovn.org/node-encap-ips:["10.0.2.7"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.7/24"} k8s.ovn.org/zone-name:ip-10-0-2-7.ec2.internal] on node ip-10-0-2-7.ec2.internal I0428 15:37:23.643131 3786 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:23.726266 3786 controller_manager.go:429] Waiting for node in zone sync took: 500.289037ms I0428 15:37:23.731289 3786 controller_manager.go:611] Switching to transit router for layer2 networks I0428 15:37:23.731304 3786 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-7.ec2.internal I0428 15:37:23.746401 3786 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 15:37:23.754039 3786 controller_manager.go:290] SCTP support detected in OVN I0428 15:37:23.754654 3786 services_controller.go:69] Creating services controller for network=default I0428 15:37:23.754758 3786 udn_enabled_svc.go:80] Waiting for service informer to sync I0428 15:37:23.754945 3786 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0428 15:37:23.755053 3786 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0428 15:37:23.755074 3786 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0428 15:37:23.755084 3786 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 15:37:23.755224 3786 udn_enabled_svc.go:95] Performing full resync I0428 15:37:23.755309 3786 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0428 15:37:23.755369 3786 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0428 15:37:23.755387 3786 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 45.235µs I0428 15:37:23.755401 3786 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 24.129µs I0428 15:37:23.755408 3786 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 38.789µs I0428 15:37:23.755414 3786 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0428 15:37:23.755427 3786 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0428 15:37:23.755441 3786 default_network_controller.go:325] Starting the default network controller I0428 15:37:23.755518 3786 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 16.227µs I0428 15:37:23.755522 3786 udn_enabled_svc.go:99] Waiting for handler to sync I0428 15:37:23.755540 3786 udn_enabled_svc.go:104] Starting worker I0428 15:37:23.765307 3786 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0428 15:37:23.765375 3786 repair.go:33] Syncing exgw routes took 54.896µs I0428 15:37:23.765401 3786 default_network_controller.go:414] Starting all the Watchers... I0428 15:37:23.765574 3786 namespace.go:98] [openshift-apiserver] adding namespace I0428 15:37:23.765580 3786 namespace.go:98] [openshift-kube-scheduler] adding namespace I0428 15:37:23.765579 3786 namespace.go:98] [openshift-backplane-csm] adding namespace I0428 15:37:23.765596 3786 namespace.go:98] [openshift-host-network] adding namespace I0428 15:37:23.765608 3786 namespace.go:98] [openshift-console-user-settings] adding namespace I0428 15:37:23.765617 3786 namespace.go:98] [openshift-dns] adding namespace I0428 15:37:23.765638 3786 namespace.go:98] [openshift-backplane] adding namespace I0428 15:37:23.765652 3786 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0428 15:37:23.765656 3786 namespace.go:98] [openshift] adding namespace I0428 15:37:23.765672 3786 namespace.go:98] [openshift-network-diagnostics] adding namespace I0428 15:37:23.765691 3786 namespace.go:98] [openshift-infra] adding namespace I0428 15:37:23.765694 3786 namespace.go:98] [openshift-marketplace] adding namespace I0428 15:37:23.766079 3786 namespace.go:98] [openshift-network-operator] adding namespace I0428 15:37:23.766108 3786 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0428 15:37:23.766162 3786 namespace.go:98] [openshift-config] adding namespace I0428 15:37:23.766341 3786 namespace.go:102] [openshift-apiserver] adding namespace took 755.05µs I0428 15:37:23.766356 3786 namespace.go:98] [openshift-authentication-operator] adding namespace I0428 15:37:23.766819 3786 namespace.go:102] [openshift-authentication-operator] adding namespace took 458.521µs I0428 15:37:23.766832 3786 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0428 15:37:23.767554 3786 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 715.916µs I0428 15:37:23.767573 3786 namespace.go:98] [openshift-operators] adding namespace I0428 15:37:23.767972 3786 namespace.go:102] [openshift-kube-scheduler] adding namespace took 2.379409ms I0428 15:37:23.767985 3786 namespace.go:98] [openshift-operators-redhat] adding namespace I0428 15:37:23.768566 3786 namespace.go:102] [openshift-backplane-csm] adding namespace took 2.968841ms I0428 15:37:23.768581 3786 namespace.go:98] [openshift-must-gather-operator] adding namespace I0428 15:37:23.769158 3786 namespace.go:102] [openshift-host-network] adding namespace took 3.555626ms I0428 15:37:23.769167 3786 namespace.go:98] [openshift-console] adding namespace I0428 15:37:23.769716 3786 namespace.go:102] [openshift-console-user-settings] adding namespace took 4.104908ms I0428 15:37:23.769728 3786 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0428 15:37:23.770163 3786 namespace.go:102] [openshift-dns] adding namespace took 4.538245ms I0428 15:37:23.770180 3786 namespace.go:98] [openshift-config-operator] adding namespace I0428 15:37:23.770732 3786 namespace.go:102] [openshift-backplane] adding namespace took 5.084721ms I0428 15:37:23.770750 3786 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0428 15:37:23.771280 3786 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 5.623807ms I0428 15:37:23.771298 3786 namespace.go:98] [openshift-monitoring] adding namespace W0428 15:37:23.771837 3786 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-89vsl: pod openshift-network-diagnostics/network-check-target-89vsl: no pod IPs found W0428 15:37:23.771852 3786 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-gzrrf: pod openshift-network-diagnostics/network-check-target-gzrrf: no pod IPs found W0428 15:37:23.771858 3786 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-w6lhl: pod openshift-network-diagnostics/network-check-target-w6lhl: no pod IPs found I0428 15:37:23.771874 3786 namespace.go:102] [openshift] adding namespace took 6.210602ms I0428 15:37:23.771888 3786 namespace.go:98] [openshift-etcd] adding namespace I0428 15:37:23.772584 3786 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.903344ms I0428 15:37:23.772603 3786 namespace.go:98] [openshift-kube-apiserver] adding namespace I0428 15:37:23.773218 3786 namespace.go:102] [openshift-infra] adding namespace took 7.514022ms I0428 15:37:23.773230 3786 namespace.go:98] [openshift-logging] adding namespace I0428 15:37:23.773721 3786 namespace.go:102] [openshift-marketplace] adding namespace took 8.011714ms I0428 15:37:23.773739 3786 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0428 15:37:23.774257 3786 namespace.go:102] [openshift-network-operator] adding namespace took 8.126523ms I0428 15:37:23.774268 3786 namespace.go:98] [openshift-console-operator] adding namespace I0428 15:37:23.774719 3786 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 8.60281ms I0428 15:37:23.774730 3786 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0428 15:37:23.775174 3786 namespace.go:102] [openshift-config] adding namespace took 9.001164ms I0428 15:37:23.775190 3786 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0428 15:37:23.775654 3786 namespace.go:102] [openshift-operators] adding namespace took 8.075287ms I0428 15:37:23.775665 3786 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0428 15:37:23.776110 3786 namespace.go:102] [openshift-operators-redhat] adding namespace took 8.115469ms I0428 15:37:23.776125 3786 namespace.go:98] [openshift-service-ca-operator] adding namespace I0428 15:37:23.776521 3786 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.93525ms I0428 15:37:23.776533 3786 namespace.go:98] [openshift-config-managed] adding namespace I0428 15:37:23.776961 3786 namespace.go:102] [openshift-console] adding namespace took 7.788265ms I0428 15:37:23.776977 3786 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0428 15:37:23.777415 3786 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 7.682221ms I0428 15:37:23.777429 3786 namespace.go:98] [openshift-ingress-operator] adding namespace I0428 15:37:23.777789 3786 namespace.go:102] [openshift-config-operator] adding namespace took 7.603868ms I0428 15:37:23.777800 3786 namespace.go:98] [openshift-customer-monitoring] adding namespace I0428 15:37:23.778260 3786 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.501236ms I0428 15:37:23.778281 3786 namespace.go:98] [openshift-ingress] adding namespace I0428 15:37:23.778774 3786 namespace.go:102] [openshift-monitoring] adding namespace took 7.467989ms I0428 15:37:23.778794 3786 namespace.go:98] [kube-system] adding namespace I0428 15:37:23.779251 3786 namespace.go:102] [openshift-etcd] adding namespace took 7.356778ms I0428 15:37:23.779267 3786 namespace.go:98] [openshift-machine-config-operator] adding namespace I0428 15:37:23.779674 3786 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.066239ms I0428 15:37:23.779688 3786 namespace.go:98] [openshift-machine-api] adding namespace I0428 15:37:23.780083 3786 namespace.go:102] [openshift-logging] adding namespace took 6.847929ms I0428 15:37:23.780117 3786 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0428 15:37:23.780513 3786 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.768909ms I0428 15:37:23.780526 3786 namespace.go:98] [kube-node-lease] adding namespace I0428 15:37:23.780899 3786 namespace.go:102] [openshift-console-operator] adding namespace took 6.625672ms I0428 15:37:23.780911 3786 namespace.go:98] [dedicated-admin] adding namespace I0428 15:37:23.781360 3786 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.6235ms I0428 15:37:23.781374 3786 namespace.go:98] [openshift-backplane-cee] adding namespace I0428 15:37:23.781907 3786 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.708194ms I0428 15:37:23.781924 3786 namespace.go:98] [openshift-image-registry] adding namespace I0428 15:37:23.782533 3786 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.861247ms I0428 15:37:23.783114 3786 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.981673ms I0428 15:37:23.783136 3786 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0428 15:37:23.783713 3786 namespace.go:102] [openshift-config-managed] adding namespace took 7.174669ms I0428 15:37:23.783727 3786 namespace.go:98] [openshift-dns-operator] adding namespace I0428 15:37:23.784230 3786 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.244349ms I0428 15:37:23.784251 3786 namespace.go:98] [openshift-route-controller-manager] adding namespace I0428 15:37:23.784794 3786 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.358498ms I0428 15:37:23.784815 3786 namespace.go:98] [openshift-backplane-cse] adding namespace I0428 15:37:23.785435 3786 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.628764ms I0428 15:37:23.785455 3786 namespace.go:98] [openshift-network-console] adding namespace I0428 15:37:23.786117 3786 namespace.go:102] [openshift-ingress] adding namespace took 7.828052ms I0428 15:37:23.786131 3786 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0428 15:37:23.786614 3786 namespace.go:102] [kube-system] adding namespace took 7.814784ms I0428 15:37:23.786628 3786 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0428 15:37:23.787066 3786 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.79261ms I0428 15:37:23.787080 3786 namespace.go:98] [openshift-insights] adding namespace I0428 15:37:23.787544 3786 namespace.go:102] [openshift-machine-api] adding namespace took 7.850908ms I0428 15:37:23.787557 3786 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0428 15:37:23.787973 3786 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.851053ms I0428 15:37:23.787986 3786 namespace.go:98] [openshift-node] adding namespace I0428 15:37:23.788397 3786 namespace.go:102] [kube-node-lease] adding namespace took 7.865515ms I0428 15:37:23.788409 3786 namespace.go:98] [kube-public] adding namespace I0428 15:37:23.788793 3786 namespace.go:102] [dedicated-admin] adding namespace took 7.877799ms I0428 15:37:23.788804 3786 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0428 15:37:23.789316 3786 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.93741ms I0428 15:37:23.789326 3786 namespace.go:98] [openshift-backplane-tam] adding namespace I0428 15:37:23.789818 3786 namespace.go:102] [openshift-image-registry] adding namespace took 7.888795ms I0428 15:37:23.789828 3786 namespace.go:98] [openshift-package-operator] adding namespace I0428 15:37:23.790246 3786 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.105184ms I0428 15:37:23.790259 3786 namespace.go:98] [openshift-ingress-canary] adding namespace I0428 15:37:23.790619 3786 namespace.go:102] [openshift-dns-operator] adding namespace took 6.887345ms I0428 15:37:23.790631 3786 namespace.go:98] [openshift-backplane-srep] adding namespace I0428 15:37:23.791144 3786 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.888188ms I0428 15:37:23.791160 3786 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0428 15:37:23.791606 3786 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.782443ms I0428 15:37:23.791625 3786 namespace.go:98] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] adding namespace I0428 15:37:23.792061 3786 namespace.go:102] [openshift-network-console] adding namespace took 6.598714ms I0428 15:37:23.792076 3786 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0428 15:37:23.792548 3786 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 6.411416ms I0428 15:37:23.792564 3786 namespace.go:98] [openshift-controller-manager] adding namespace I0428 15:37:23.792915 3786 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.280148ms I0428 15:37:23.792933 3786 namespace.go:98] [default] adding namespace I0428 15:37:23.793400 3786 namespace.go:102] [openshift-insights] adding namespace took 6.314477ms I0428 15:37:23.793416 3786 namespace.go:98] [openshift-network-node-identity] adding namespace I0428 15:37:23.793883 3786 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.311537ms I0428 15:37:23.793901 3786 namespace.go:98] [openshift-authentication] adding namespace I0428 15:37:23.794399 3786 namespace.go:102] [openshift-node] adding namespace took 6.407802ms I0428 15:37:23.794415 3786 namespace.go:98] [openshift-multus] adding namespace I0428 15:37:23.794921 3786 namespace.go:102] [kube-public] adding namespace took 6.506552ms I0428 15:37:23.794938 3786 namespace.go:98] [openshift-apiserver-operator] adding namespace I0428 15:37:23.795403 3786 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.591824ms I0428 15:37:23.795422 3786 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0428 15:37:23.796350 3786 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.017952ms I0428 15:37:23.796366 3786 namespace.go:98] [openshift-backplane-mobb] adding namespace I0428 15:37:23.796843 3786 namespace.go:102] [openshift-package-operator] adding namespace took 7.00993ms I0428 15:37:23.796859 3786 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0428 15:37:23.797374 3786 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.110155ms I0428 15:37:23.797856 3786 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.220394ms I0428 15:37:23.798334 3786 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.168327ms I0428 15:37:23.798808 3786 namespace.go:102] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] adding namespace took 7.177491ms I0428 15:37:23.799358 3786 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 7.275823ms I0428 15:37:23.799814 3786 namespace.go:102] [openshift-controller-manager] adding namespace took 7.244261ms I0428 15:37:23.800237 3786 namespace.go:102] [default] adding namespace took 7.299643ms I0428 15:37:23.800251 3786 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0428 15:37:23.800697 3786 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.275206ms I0428 15:37:23.800712 3786 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace W0428 15:37:23.801122 3786 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-c85fc: pod openshift-multus/network-metrics-daemon-c85fc: no pod IPs found W0428 15:37:23.801137 3786 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-xdscj: pod openshift-multus/network-metrics-daemon-xdscj: no pod IPs found I0428 15:37:23.801138 3786 namespace.go:102] [openshift-authentication] adding namespace took 7.230781ms W0428 15:37:23.801145 3786 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-tvj87: pod openshift-multus/network-metrics-daemon-tvj87: no pod IPs found I0428 15:37:23.801148 3786 namespace.go:98] [openshift-cluster-version] adding namespace I0428 15:37:23.801632 3786 namespace.go:102] [openshift-multus] adding namespace took 7.211737ms I0428 15:37:23.802114 3786 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.169813ms I0428 15:37:23.802136 3786 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0428 15:37:23.802569 3786 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.141851ms I0428 15:37:23.802959 3786 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.588257ms I0428 15:37:23.803400 3786 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 6.536584ms I0428 15:37:23.803868 3786 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 3.61126ms I0428 15:37:23.804336 3786 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 3.618929ms I0428 15:37:23.804775 3786 namespace.go:102] [openshift-cluster-version] adding namespace took 3.621833ms I0428 15:37:23.805119 3786 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 2.977939ms W0428 15:37:23.805199 3786 master.go:471] Unable to parse remote node ip-10-0-0-9.ec2.internal chassis-id annotation. Chassis may be removed during sync W0428 15:37:23.805206 3786 master.go:471] Unable to parse remote node ip-10-0-1-106.ec2.internal chassis-id annotation. Chassis may be removed during sync I0428 15:37:23.805315 3786 master.go:548] Adding or Updating local node "ip-10-0-2-7.ec2.internal" for network "default" I0428 15:37:23.805358 3786 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-9.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-9.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-9.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-9.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-9.ec2.internal I0428 15:37:23.805369 3786 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-106.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-106.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-106.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-106.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-106.ec2.internal W0428 15:37:23.813547 3786 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-tvj87: pod openshift-multus/network-metrics-daemon-tvj87: no pod IPs found W0428 15:37:23.813566 3786 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-89vsl: pod openshift-network-diagnostics/network-check-target-89vsl: no pod IPs found I0428 15:37:23.814584 3786 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-7.ec2.internal for the network default I0428 15:37:23.814959 3786 zone_ic_handler.go:195] Time taken to create transit switch: 360.286µs I0428 15:37:23.816181 3786 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0428 15:37:23.816210 3786 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0428 15:37:23.816223 3786 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0428 15:37:23.816253 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-9.ec2.internal in network "default" I0428 15:37:23.816287 3786 node_tracker.go:235] Node ip-10-0-0-9.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-9.ec2.internal" I0428 15:37:23.816302 3786 node_tracker.go:169] Node ip-10-0-0-9.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:23.816317 3786 services_controller.go:574] Full service sync requested for network=default I0428 15:37:23.816327 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:23.816336 3786 node_tracker.go:235] Node ip-10-0-1-106.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-106.ec2.internal" I0428 15:37:23.816342 3786 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:23.816346 3786 services_controller.go:574] Full service sync requested for network=default I0428 15:37:23.816351 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-7.ec2.internal in network "default" I0428 15:37:23.816378 3786 node_tracker.go:169] Node ip-10-0-2-7.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:23.816385 3786 services_controller.go:574] Full service sync requested for network=default I0428 15:37:23.916545 3786 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0428 15:37:23.916571 3786 services_controller.go:208] Setting up event handlers for services for network=default I0428 15:37:23.916637 3786 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0428 15:37:23.916687 3786 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0428 15:37:23.916704 3786 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0428 15:37:24.016784 3786 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0428 15:37:24.016805 3786 repair.go:60] Starting repairing loop for services I0428 15:37:24.016885 3786 repair.go:131] Deleted 0 stale service LBs I0428 15:37:24.016904 3786 repair.go:137] Deleted 0 stale Chassis Template Vars I0428 15:37:24.016919 3786 repair.go:62] Finished repairing loop for services: 116.214µs I0428 15:37:24.016946 3786 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0428 15:37:24.016955 3786 services_controller.go:254] Starting workers for network=default I0428 15:37:24.017346 3786 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-w6lhl, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-w6lhl: suppressed error logged: pod openshift-network-diagnostics/network-check-target-w6lhl: no pod IPs found I0428 15:37:24.017380 3786 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-xdscj, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-xdscj: suppressed error logged: pod openshift-multus/network-metrics-daemon-xdscj: no pod IPs found I0428 15:37:24.017409 3786 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-gzrrf, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-gzrrf: suppressed error logged: pod openshift-network-diagnostics/network-check-target-gzrrf: no pod IPs found I0428 15:37:24.017421 3786 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-tvj87] creating logical port openshift-multus_network-metrics-daemon-tvj87 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:24.017427 3786 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-89vsl] creating logical port openshift-network-diagnostics_network-check-target-89vsl for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:24.017449 3786 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-c85fc, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-c85fc: suppressed error logged: pod openshift-multus/network-metrics-daemon-c85fc: no pod IPs found I0428 15:37:24.017620 3786 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-89vsl I0428 15:37:24.017685 3786 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-tvj87 I0428 15:37:24.034734 3786 pod.go:62] [openshift-multus/network-metrics-daemon-tvj87] pod update took 17.231736ms I0428 15:37:24.034752 3786 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-tvj87] addLogicalPort annotation time took 17.253995ms I0428 15:37:24.035655 3786 pods.go:271] [openshift-multus/network-metrics-daemon-tvj87] addLogicalPort took 18.248002ms, libovsdb time 673.782µs I0428 15:37:24.039474 3786 pod.go:62] [openshift-network-diagnostics/network-check-target-89vsl] pod update took 21.951574ms I0428 15:37:24.039498 3786 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-89vsl] addLogicalPort annotation time took 21.995851ms I0428 15:37:24.040467 3786 pods.go:271] [openshift-network-diagnostics/network-check-target-89vsl] addLogicalPort took 23.04942ms, libovsdb time 757.813µs I0428 15:37:24.040833 3786 admin_network_policy_controller.go:227] Starting controller default-network-controller I0428 15:37:24.040860 3786 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0428 15:37:24.040882 3786 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0428 15:37:24.040889 3786 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0428 15:37:24.041154 3786 repair.go:29] Repairing admin network policies took 258.601µs I0428 15:37:24.041316 3786 repair.go:92] Repairing baseline admin network policies took 149.326µs I0428 15:37:24.041612 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0428 15:37:24.042537 3786 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0428 15:37:24.042558 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 15:37:24.044441 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0428 15:37:24.044462 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0428 15:37:24.044469 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0428 15:37:24.044825 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0428 15:37:24.044852 3786 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0428 15:37:24.044862 3786 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0428 15:37:24.046228 3786 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0428 15:37:24.046258 3786 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0428 15:37:24.046265 3786 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0428 15:37:24.047836 3786 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0428 15:37:24.047861 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0428 15:37:24.047871 3786 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0428 15:37:24.049495 3786 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0428 15:37:24.049516 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0428 15:37:24.049546 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 15:37:24.051528 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0428 15:37:24.051556 3786 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0428 15:37:24.051568 3786 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0428 15:37:24.053616 3786 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0428 15:37:24.053635 3786 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0428 15:37:24.053644 3786 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0428 15:37:24.054724 3786 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0428 15:37:24.054744 3786 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0428 15:37:24.054756 3786 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0428 15:37:24.056037 3786 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0428 15:37:24.056056 3786 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0428 15:37:24.056064 3786 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0428 15:37:24.057416 3786 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0428 15:37:24.057439 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0428 15:37:24.057448 3786 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0428 15:37:24.058177 3786 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0428 15:37:24.058196 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0428 15:37:24.058206 3786 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0428 15:37:24.058775 3786 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0428 15:37:24.058792 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0428 15:37:24.058799 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0428 15:37:24.059315 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0428 15:37:24.059335 3786 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0428 15:37:24.059343 3786 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0428 15:37:24.059881 3786 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0428 15:37:24.059904 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0428 15:37:24.059914 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0428 15:37:24.060523 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0428 15:37:24.060545 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0428 15:37:24.060558 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0428 15:37:24.061126 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0428 15:37:24.061150 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0428 15:37:24.061160 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0428 15:37:24.061705 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0428 15:37:24.061723 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0428 15:37:24.061731 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0428 15:37:24.062250 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0428 15:37:24.062275 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0428 15:37:24.062284 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0428 15:37:24.064017 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0428 15:37:24.064036 3786 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0428 15:37:24.064043 3786 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0428 15:37:24.065684 3786 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0428 15:37:24.065702 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0428 15:37:24.065710 3786 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0428 15:37:24.066178 3786 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0428 15:37:24.066193 3786 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0428 15:37:24.066203 3786 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0428 15:37:24.067473 3786 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0428 15:37:24.067492 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0428 15:37:24.067499 3786 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0428 15:37:24.067978 3786 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0428 15:37:24.067996 3786 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0428 15:37:24.068003 3786 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0428 15:37:24.069254 3786 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0428 15:37:24.069272 3786 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0428 15:37:24.069282 3786 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0428 15:37:24.070346 3786 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0428 15:37:24.070372 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0428 15:37:24.070390 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0428 15:37:24.071025 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0428 15:37:24.071044 3786 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0428 15:37:24.071054 3786 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0428 15:37:24.072198 3786 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0428 15:37:24.072218 3786 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0428 15:37:24.072229 3786 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0428 15:37:24.073913 3786 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0428 15:37:24.073932 3786 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0428 15:37:24.073944 3786 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0428 15:37:24.074536 3786 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0428 15:37:24.080053 3786 egressfirewall.go:374] Starting EgressFirewall controller I0428 15:37:24.080116 3786 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0428 15:37:24.080120 3786 controller.go:133] Adding controller egress-firewall-controller event handlers I0428 15:37:24.080150 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0428 15:37:24.080164 3786 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0428 15:37:24.080151 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0428 15:37:24.080172 3786 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0428 15:37:24.080435 3786 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0428 15:37:24.080448 3786 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0428 15:37:24.080454 3786 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0428 15:37:24.080462 3786 egressqos.go:193] Setting up event handlers for EgressQoS I0428 15:37:24.080481 3786 egressfirewall.go:723] Syncing node "ip-10-0-1-106.ec2.internal" for egress firewall I0428 15:37:24.080494 3786 egressfirewall.go:723] Syncing node "ip-10-0-2-7.ec2.internal" for egress firewall I0428 15:37:24.080501 3786 egressfirewall.go:723] Syncing node "ip-10-0-0-9.ec2.internal" for egress firewall I0428 15:37:24.080593 3786 egressqos.go:245] Starting EgressQoS Controller I0428 15:37:24.080610 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0428 15:37:24.080619 3786 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0428 15:37:24.080627 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0428 15:37:24.080631 3786 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0428 15:37:24.080636 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0428 15:37:24.080640 3786 shared_informer.go:356] "Caches are synced" controller="egressqos" I0428 15:37:24.080644 3786 egressqos.go:259] Repairing EgressQoSes I0428 15:37:24.080647 3786 egressqos.go:399] Starting repairing loop for egressqos I0428 15:37:24.080725 3786 egressqos.go:401] Finished repairing loop for egressqos: 77.77µs I0428 15:37:24.080747 3786 egressservice_zone.go:132] Setting up event handlers for Egress Services I0428 15:37:24.080792 3786 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-7.ec2.internal I0428 15:37:24.080840 3786 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-7.ec2.internal : 50.422µs I0428 15:37:24.080979 3786 egressservice_zone.go:210] Starting Egress Services Controller I0428 15:37:24.080990 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 15:37:24.080998 3786 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 15:37:24.081006 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 15:37:24.081011 3786 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 15:37:24.081016 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 15:37:24.081020 3786 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 15:37:24.081027 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0428 15:37:24.081031 3786 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0428 15:37:24.081035 3786 egressservice_zone.go:228] Repairing Egress Services I0428 15:37:24.081336 3786 master_controller.go:88] Starting Admin Policy Based Route Controller I0428 15:37:24.081350 3786 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 15:37:24.081401 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:37:24.081412 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 11.961µs I0428 15:37:24.081420 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:37:24.081424 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 4.728µs I0428 15:37:24.081429 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-7.ec2.internal I0428 15:37:24.081433 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-7.ec2.internal: 4.918µs I0428 15:37:24.081475 3786 default_network_controller.go:585] Completing all the Watchers took 316.061637ms I0428 15:37:24.081495 3786 default_network_controller.go:589] Starting unidling controllers I0428 15:37:24.081510 3786 unidle.go:46] Registering OVN SB ControllerEvent handler I0428 15:37:24.081521 3786 unidle.go:63] Populating Initial ContollerEvent events I0428 15:37:24.081534 3786 unidle.go:79] Setting up event handlers for services I0428 15:37:24.081604 3786 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0428 15:37:24.143658 3786 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:24.644245 3786 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:25.143678 3786 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:25.242486 3786 default_network_controller.go:1025] Node "ip-10-0-0-9.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:25.643737 3786 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:25.892668 3786 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0428 15:37:26.048843 3786 egressfirewall.go:723] Syncing node "ip-10-0-0-9.ec2.internal" for egress firewall I0428 15:37:26.048884 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:37:26.048893 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 10.741µs I0428 15:37:26.048896 3786 default_network_controller.go:1025] Node "ip-10-0-0-9.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:26.143746 3786 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:26.162333 3786 bridgeconfig.go:537] Gateway is ready I0428 15:37:26.162356 3786 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0428 15:37:26.175475 3786 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.7. Updating OVN encap IP. I0428 15:37:26.181627 3786 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.7 - it is already configured I0428 15:37:26.181643 3786 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.7/24:{} 172.20.0.1/32:{}]. Updating annotations. I0428 15:37:26.181860 3786 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.7/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-7.ec2.internal","mac-address":"0e:99:b9:48:ab:0b","ip-addresses":["10.0.2.7/24"],"ip-address":"10.0.2.7/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:2947311f-9ee9-4ee8-800d-ae9d29b50ea8 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.7/24"}] on node ip-10-0-2-7.ec2.internal I0428 15:37:26.199204 3786 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.7 I0428 15:37:26.199304 3786 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-7.ec2.internal I0428 15:37:26.200041 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-7.ec2.internal in network "default" I0428 15:37:26.200071 3786 master.go:548] Adding or Updating local node "ip-10-0-2-7.ec2.internal" for network "default" I0428 15:37:26.200114 3786 egressfirewall.go:723] Syncing node "ip-10-0-2-7.ec2.internal" for egress firewall I0428 15:37:26.200121 3786 node_tracker.go:169] Node ip-10-0-2-7.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:26.200137 3786 services_controller.go:574] Full service sync requested for network=default I0428 15:37:26.200138 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-7.ec2.internal I0428 15:37:26.200154 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-7.ec2.internal: 20.597µs I0428 15:37:26.205615 3786 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-7.ec2.internal. Updating no re-route policies I0428 15:37:26.205682 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-7.ec2.internal I0428 15:37:26.205706 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-7.ec2.internal I0428 15:37:26.205716 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-c85fc I0428 15:37:26.205723 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-r2wjt I0428 15:37:26.205741 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-7.ec2.internal after 0 failed attempt(s) I0428 15:37:26.205750 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-r2wjt I0428 15:37:26.205746 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-t5fvn I0428 15:37:26.205753 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-sgh5j I0428 15:37:26.205758 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-xdscj I0428 15:37:26.205765 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-sgh5j I0428 15:37:26.205764 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-w6lhl I0428 15:37:26.205772 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-xdscj I0428 15:37:26.205771 3786 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-2zrgg I0428 15:37:26.205779 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-w6lhl I0428 15:37:26.205778 3786 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-7.ec2.internal I0428 15:37:26.205786 3786 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-2zrgg I0428 15:37:26.205781 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-kw46n I0428 15:37:26.205793 3786 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-7.ec2.internal I0428 15:37:26.205795 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-kw46n I0428 15:37:26.205806 3786 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-2zrgg after 0 failed attempt(s) I0428 15:37:26.205792 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-89vsl I0428 15:37:26.205811 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-kw46n after 0 failed attempt(s) I0428 15:37:26.205809 3786 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-xdscj, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-xdscj: suppressed error logged: pod openshift-multus/network-metrics-daemon-xdscj: no pod IPs found I0428 15:37:26.205816 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-sgh5j after 0 failed attempt(s) I0428 15:37:26.205820 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-89vsl I0428 15:37:26.205770 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-r2wjt after 0 failed attempt(s) I0428 15:37:26.205825 3786 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-w6lhl, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-w6lhl: suppressed error logged: pod openshift-network-diagnostics/network-check-target-w6lhl: no pod IPs found I0428 15:37:26.205765 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-t5fvn I0428 15:37:26.205816 3786 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-7.ec2.internal after 0 failed attempt(s) I0428 15:37:26.205848 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-t5fvn after 0 failed attempt(s) I0428 15:37:26.205846 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-7q6jg I0428 15:37:26.205738 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-c85fc I0428 15:37:26.205861 3786 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-89vsl] creating logical port openshift-network-diagnostics_network-check-target-89vsl for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:26.205867 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-7q6jg I0428 15:37:26.205878 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-gzrrf I0428 15:37:26.205888 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-gzrrf I0428 15:37:26.205890 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-z7gv9 I0428 15:37:26.205890 3786 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-c85fc, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-c85fc: suppressed error logged: pod openshift-multus/network-metrics-daemon-c85fc: no pod IPs found I0428 15:37:26.205895 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-tvj87 I0428 15:37:26.205902 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-z7gv9 I0428 15:37:26.205910 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-tvj87 I0428 15:37:26.205910 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-7wlsb I0428 15:37:26.205923 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-z7gv9 after 0 failed attempt(s) I0428 15:37:26.205921 3786 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-gzrrf, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-gzrrf: suppressed error logged: pod openshift-network-diagnostics/network-check-target-gzrrf: no pod IPs found I0428 15:37:26.205924 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-7wlsb I0428 15:37:26.205939 3786 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-tvj87] creating logical port openshift-multus_network-metrics-daemon-tvj87 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:26.205953 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-7wlsb after 0 failed attempt(s) I0428 15:37:26.205868 3786 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-n5xk6 I0428 15:37:26.205974 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-7q6jg after 0 failed attempt(s) I0428 15:37:26.205983 3786 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-n5xk6 I0428 15:37:26.206009 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-n5xk6 after 0 failed attempt(s) I0428 15:37:26.206717 3786 pods.go:271] [openshift-network-diagnostics/network-check-target-89vsl] addLogicalPort took 868.505µs, libovsdb time 331.015µs I0428 15:37:26.206735 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-89vsl after 0 failed attempt(s) I0428 15:37:26.207183 3786 pods.go:271] [openshift-multus/network-metrics-daemon-tvj87] addLogicalPort took 1.251474ms, libovsdb time 695.462µs I0428 15:37:26.207200 3786 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-tvj87 after 0 failed attempt(s) I0428 15:37:26.217893 3786 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0428 15:37:26.231817 3786 iptables.go:108] Creating table: mangle chain: OUTPUT I0428 15:37:26.235491 3786 iptables.go:108] Creating table: nat chain: OUTPUT I0428 15:37:26.285435 3786 iptables.go:108] Creating table: nat chain: PREROUTING I0428 15:37:26.343589 3786 iptables.go:108] Creating table: filter chain: FORWARD I0428 15:37:26.362941 3786 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0428 15:37:26.363058 3786 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0428 15:37:26.363152 3786 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0428 15:37:26.363247 3786 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0428 15:37:26.363337 3786 gateway_init.go:325] Gateway and management port readiness took 2.724454733s I0428 15:37:26.363368 3786 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0428 15:37:26.363404 3786 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0428 15:37:26.363417 3786 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0428 15:37:26.363426 3786 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 15:37:26.363500 3786 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0428 15:37:26.363593 3786 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 57.346µs I0428 15:37:26.363913 3786 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0428 15:37:26.363940 3786 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.927µs I0428 15:37:26.363949 3786 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 36.497µs I0428 15:37:26.363955 3786 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0428 15:37:26.363965 3786 nad_controller.go:379] [node-nad-controller NAD controller]: started I0428 15:37:26.363972 3786 default_node_network_controller.go:899] Starting the default node network controller I0428 15:37:26.363977 3786 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0428 15:37:26.364061 3786 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.367µs I0428 15:37:26.364184 3786 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0428 15:37:26.364194 3786 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0428 15:37:26.364221 3786 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1d240 0xc0002bf960 0xc0002bf9d0 0xc0002bfa40 0xc0002bfab0 0xc0002bfb20 0xc0002bfb90 0xc000b2b680 0xc0002bfc00 0xc0002bfc70 0xc00022e2d0 0xc0064455e0 0xc0002bfce0 map[0x2ff9080:0xc00279d2c0 0x2ff95c0:0xc00279d440 0x2ffa2e0:0xc00279d200 0x2ffa580:0xc00279d260 0x2ffad60:0xc00279d320 0x2ffb000:0xc00279d380 0x2ffb2a0:0xc00279d3e0 0x2ffba80:0xc00279d4a0 0x2ffbd20:0xc00279d500 0x3026980:0xc0008d9140 0x3026d20:0xc0008d91a0 0x3027800:0xc000c99680 0x3029160:0xc001b3e000 0x30298a0:0xc0008d9380 0x302a720:0xc0008d90e0] 0xc000b20690 0} I0428 15:37:26.364300 3786 gateway.go:151] Starting gateway service sync I0428 15:37:26.364832 3786 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0428 15:37:26.367396 3786 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0428 15:37:26.389004 3786 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0428 15:37:26.390327 3786 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0428 15:37:26.391607 3786 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0428 15:37:26.426000 3786 gateway.go:168] Gateway service sync done. Time taken: 61.685056ms I0428 15:37:26.426554 3786 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30931 I0428 15:37:26.426825 3786 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1d240 0xc0002bf960 0xc0002bf9d0 0xc0002bfa40 0xc0002bfab0 0xc0002bfb20 0xc0002bfb90 0xc000b2b680 0xc0002bfc00 0xc0002bfc70 0xc00022e2d0 0xc0064455e0 0xc0002bfce0 map[0x2ff9080:0xc00279d2c0 0x2ff95c0:0xc00279d440 0x2ffa2e0:0xc00279d200 0x2ffa580:0xc00279d260 0x2ffad60:0xc00279d320 0x2ffb000:0xc00279d380 0x2ffb2a0:0xc00279d3e0 0x2ffba80:0xc00279d4a0 0x2ffbd20:0xc00279d500 0x3026980:0xc0008d9140 0x3026d20:0xc0008d91a0 0x3027800:0xc000c99680 0x3029160:0xc001b3e000 0x30298a0:0xc0008d9380 0x302a720:0xc0008d90e0] 0xc000b20690 0} I0428 15:37:26.427311 3786 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b1d240 0xc0002bf960 0xc0002bf9d0 0xc0002bfa40 0xc0002bfab0 0xc0002bfb20 0xc0002bfb90 0xc000b2b680 0xc0002bfc00 0xc0002bfc70 0xc00022e2d0 0xc0064455e0 0xc0002bfce0 map[0x2ff9080:0xc00279d2c0 0x2ff95c0:0xc00279d440 0x2ffa2e0:0xc00279d200 0x2ffa580:0xc00279d260 0x2ffad60:0xc00279d320 0x2ffb000:0xc00279d380 0x2ffb2a0:0xc00279d3e0 0x2ffba80:0xc00279d4a0 0x2ffbd20:0xc00279d500 0x3026980:0xc0008d9140 0x3026d20:0xc0008d91a0 0x3027800:0xc000c99680 0x3029160:0xc001b3e000 0x30298a0:0xc0008d9380 0x302a720:0xc0008d90e0] 0xc000b20690 0} I0428 15:37:26.427350 3786 gateway.go:532] Reconciling gateway with updates I0428 15:37:26.432612 3786 gateway.go:339] Spawning Conntrack Rule Check Thread I0428 15:37:26.432716 3786 default_node_network_controller.go:944] Gateway and management port readiness took 68.513345ms I0428 15:37:26.432719 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:26.432738 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:26.432755 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:26.432776 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:26.432771 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 15:37:26.432782 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 15:37:26.432776 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:26.432792 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 15:37:26.432790 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:26.432799 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:26.432803 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:26.432813 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:26.432821 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:26.432820 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:26.432837 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:26.432834 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:26.432855 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 15:37:26.432861 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:26.432860 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:26.432866 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 15:37:26.432807 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 15:37:26.432877 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 15:37:26.432878 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:26.432891 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:26.432893 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:26.432887 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:26.432890 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:26.432897 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:26.432907 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:26.432906 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:26.432916 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:26.432916 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:26.432922 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:26.432934 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:26.432935 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:26.432840 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:26.432960 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:26.432891 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:26.432919 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:26.432987 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:26.432997 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:26.432870 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:26.433039 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:26.432870 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 15:37:26.432897 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:26.433007 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 15:37:26.433064 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:26.432877 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:26.433117 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:26.433127 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:26.433142 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:26.433149 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:26.433158 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:26.433166 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:26.433160 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:26.432916 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:26.433172 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:26.433279 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:26.433173 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 15:37:26.433182 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:26.433294 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:26.433189 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:26.433306 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 15:37:26.433309 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:26.433316 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:26.433316 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:26.433324 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 15:37:26.433328 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:26.433340 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:26.433343 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:26.433346 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:26.433361 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:26.433363 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:26.433368 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:26.433371 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 15:37:26.433371 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:26.433200 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:26.433389 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:26.433279 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:26.433397 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:26.433402 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:26.433371 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 15:37:26.433411 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 15:37:26.433403 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:26.433384 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:26.433434 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:26.433324 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 15:37:26.433308 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:26.433456 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:26.433160 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:26.433302 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:26.433326 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:26.433527 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:26.433535 3786 port_claim.go:71] Svc openshift-ingress/router-default: port 32194 is already open, no action needed I0428 15:37:26.433547 3786 port_claim.go:71] Svc openshift-ingress/router-default: port 31728 is already open, no action needed I0428 15:37:26.433197 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 15:37:26.433568 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 15:37:26.433354 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 15:37:26.433386 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:26.433181 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:26.433373 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:26.433643 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:26.433646 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:26.435018 3786 node_ip_handler_linux.go:160] Node IP manager is running W0428 15:37:26.477581 3786 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 15:37:26.477748 3786 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 15:37:26.477925 3786 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0428 15:37:26.486352 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 15:37:26.486388 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 15:37:26.486399 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 15:37:26.486405 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 15:37:26.486411 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 15:37:26.486417 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 15:37:26.486435 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 15:37:26.486451 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 15:37:26.486474 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 15:37:26.486484 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 15:37:26.486489 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 15:37:26.486495 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 15:37:26.486500 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 15:37:26.486507 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 15:37:26.486512 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 15:37:26.486517 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 15:37:26.486524 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 15:37:26.486530 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 15:37:26.486535 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 15:37:26.486541 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 15:37:26.486547 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 15:37:26.486553 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 15:37:26.486575 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 15:37:26.486584 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 15:37:26.486589 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 15:37:26.486594 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 15:37:26.531141 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 15:37:26.531168 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 15:37:26.570590 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 15:37:26.570603 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 15:37:26.570633 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 15:37:27.994673 3786 default_node_network_controller.go:1380] Starting node controller node sync W0428 15:37:27.994715 3786 egressip_healthcheck.go:76] Health checking using insecure connection I0428 15:37:27.994759 3786 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0428 15:37:28.007137 3786 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 12.442966ms I0428 15:37:28.007285 3786 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0428 15:37:28.007619 3786 egressservice_node.go:93] Setting up event handlers for Egress Services I0428 15:37:28.007732 3786 egressservice_node.go:183] Starting Egress Services Controller I0428 15:37:28.007747 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 15:37:28.007759 3786 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 15:37:28.007764 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 15:37:28.007770 3786 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 15:37:28.007776 3786 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 15:37:28.007781 3786 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 15:37:28.007785 3786 egressservice_node.go:197] Repairing Egress Services I0428 15:37:28.018745 3786 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0428 15:37:28.018758 3786 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 15:37:28.018913 3786 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0428 15:37:28.018925 3786 link_network_manager.go:119] Link manager is running I0428 15:37:28.018934 3786 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0428 15:37:28.019128 3786 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0428 15:37:28.019352 3786 vrf_manager.go:128] VRF manager is running I0428 15:37:28.019563 3786 node_controller_manager.go:437] Removing flows to drop GARP I0428 15:37:28.019576 3786 gateway.go:532] Reconciling gateway with updates I0428 15:37:28.045418 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:28.045443 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:28.045448 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:28.045453 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:28.045459 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:28.045461 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:28.045468 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 15:37:28.045465 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 15:37:28.045474 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:28.045473 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:28.045479 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:28.045489 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:28.045489 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:28.045494 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:28.045496 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:28.045486 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:28.045499 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:28.045507 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:28.045504 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:28.045514 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:28.045515 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:28.045515 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:28.045520 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:28.045527 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:28.045530 3786 port_claim.go:71] Svc openshift-ingress/router-default: port 32194 is already open, no action needed I0428 15:37:28.045534 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:28.045510 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:28.045543 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 15:37:28.045540 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 15:37:28.045500 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:28.045553 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:28.045566 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 15:37:28.045577 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 15:37:28.045576 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:28.045580 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:28.045581 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:28.045581 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:28.045587 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:28.045590 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:28.045588 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:28.045595 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:28.045587 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:28.045603 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:28.045602 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:28.045610 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:28.045445 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:28.045612 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:28.045614 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:28.045618 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 15:37:28.045620 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:28.045620 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:28.045624 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:28.045624 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:28.045631 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:28.045636 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:28.045608 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:28.045668 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:28.045578 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:28.045707 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:28.045463 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:28.045526 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:28.045566 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 15:37:28.045536 3786 port_claim.go:71] Svc openshift-ingress/router-default: port 31728 is already open, no action needed I0428 15:37:28.045485 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 15:37:28.045433 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:28.045843 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:28.045425 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:28.045867 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:28.045473 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:28.045425 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:28.045914 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:28.045481 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:28.045939 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:28.045486 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:28.045962 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:28.045489 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:28.045501 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:28.045501 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:28.046024 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:28.045508 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:28.045544 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:28.046061 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:28.045548 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 15:37:28.045567 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:28.046085 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:28.045585 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 15:37:28.045593 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:28.046127 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 15:37:28.045592 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:28.046134 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:28.045601 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:28.046155 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:28.045597 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:28.045598 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 15:37:28.045611 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 15:37:28.045612 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:28.046199 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:28.045611 3786 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 15:37:28.046234 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 15:37:28.045618 3786 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:28.069593 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 15:37:28.069622 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 15:37:28.069630 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 15:37:28.069637 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 15:37:28.069643 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 15:37:28.069649 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 15:37:28.069655 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 15:37:28.069662 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 15:37:28.069667 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 15:37:28.069673 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 15:37:28.069677 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 15:37:28.069684 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 15:37:28.069689 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 15:37:28.069694 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 15:37:28.109699 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 15:37:28.109719 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 15:37:28.109739 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 15:37:28.109747 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 15:37:28.109755 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 15:37:28.109766 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 15:37:28.109773 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 15:37:28.109779 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 15:37:28.109786 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 15:37:28.109792 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 15:37:28.109797 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 15:37:28.109802 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 15:37:28.109807 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 15:37:28.109813 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 15:37:28.109818 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 15:37:28.109823 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 15:37:28.109828 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 15:37:28.109834 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 15:37:28.109838 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 15:37:28.109847 3786 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 15:37:30.024989 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-7.ec2.internal I0428 15:37:30.025020 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-7.ec2.internal: 38.627µs I0428 15:37:30.050468 3786 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7fc7c49d95-dgxkp] creating logical port openshift-image-registry_image-registry-7fc7c49d95-dgxkp for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.050702 3786 kube.go:256] Updating pod openshift-image-registry/image-registry-7fc7c49d95-dgxkp I0428 15:37:30.071489 3786 pod.go:62] [openshift-image-registry/image-registry-7fc7c49d95-dgxkp] pod update took 20.832928ms I0428 15:37:30.071517 3786 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7fc7c49d95-dgxkp] addLogicalPort annotation time took 20.865784ms I0428 15:37:30.072607 3786 pods.go:271] [openshift-image-registry/image-registry-7fc7c49d95-dgxkp] addLogicalPort took 22.16008ms, libovsdb time 850.728µs I0428 15:37:30.100221 3786 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-2b6k5] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-2b6k5 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.100339 3786 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-2b6k5 I0428 15:37:30.108418 3786 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-9vnss] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-9vnss for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.108502 3786 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-n6p49] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-n6p49 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.108529 3786 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-9vnss I0428 15:37:30.108593 3786 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-n6p49 I0428 15:37:30.112387 3786 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-v566m] creating logical port openshift-insights_insights-operator-6dbb575d6c-v566m for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.112447 3786 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-6d4c95c6c5-nsvcp] creating logical port openshift-ingress_router-default-6d4c95c6c5-nsvcp for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.112487 3786 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-v566m I0428 15:37:30.112529 3786 kube.go:256] Updating pod openshift-ingress/router-default-6d4c95c6c5-nsvcp I0428 15:37:30.112584 3786 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-fwx5l] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-fwx5l for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.112728 3786 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-fwx5l I0428 15:37:30.140478 3786 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-n6p49] pod update took 31.90835ms I0428 15:37:30.140509 3786 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-n6p49] addLogicalPort annotation time took 31.942026ms I0428 15:37:30.140598 3786 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-9vnss] pod update took 32.08712ms I0428 15:37:30.140624 3786 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-9vnss] addLogicalPort annotation time took 32.117406ms I0428 15:37:30.140657 3786 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-2b6k5] pod update took 40.342357ms I0428 15:37:30.140678 3786 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-2b6k5] addLogicalPort annotation time took 40.366951ms I0428 15:37:30.141974 3786 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-n6p49] addLogicalPort took 33.478708ms, libovsdb time 1.132236ms I0428 15:37:30.142246 3786 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-9vnss] addLogicalPort took 33.844862ms, libovsdb time 1.351651ms I0428 15:37:30.142446 3786 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-2b6k5] addLogicalPort took 42.246386ms, libovsdb time 1.531416ms I0428 15:37:30.144234 3786 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-v566m] pod update took 31.771575ms I0428 15:37:30.144254 3786 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-v566m] addLogicalPort annotation time took 31.789855ms I0428 15:37:30.145248 3786 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-v566m] addLogicalPort took 32.876395ms, libovsdb time 782.201µs I0428 15:37:30.154429 3786 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-fwx5l] pod update took 41.720963ms I0428 15:37:30.154453 3786 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-fwx5l] addLogicalPort annotation time took 41.746453ms I0428 15:37:30.155577 3786 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-fwx5l] addLogicalPort took 43.004729ms, libovsdb time 887.551µs I0428 15:37:30.162609 3786 pod.go:62] [openshift-ingress/router-default-6d4c95c6c5-nsvcp] pod update took 50.10208ms I0428 15:37:30.162635 3786 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-6d4c95c6c5-nsvcp] addLogicalPort annotation time took 50.128339ms I0428 15:37:30.163051 3786 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-67wp8] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-67wp8 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.163201 3786 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-67wp8 I0428 15:37:30.163693 3786 pods.go:271] [openshift-ingress/router-default-6d4c95c6c5-nsvcp] addLogicalPort took 51.259082ms, libovsdb time 778.43µs I0428 15:37:30.166470 3786 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-tdmlr] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-tdmlr for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.166527 3786 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-s7fh4] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-s7fh4 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.166591 3786 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-tdmlr I0428 15:37:30.166611 3786 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-s7fh4 I0428 15:37:30.171476 3786 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-qnplk] creating logical port openshift-ingress-canary_ingress-canary-qnplk for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.171519 3786 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-cttkb] creating logical port openshift-network-diagnostics_network-check-source-84fd6d9d54-cttkb for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.171564 3786 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-qnplk I0428 15:37:30.171583 3786 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-bkn49] creating logical port openshift-console-operator_console-operator-66dbbb6468-bkn49 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.171594 3786 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-84fd6d9d54-cttkb I0428 15:37:30.171639 3786 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-bkn49 I0428 15:37:30.171850 3786 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-dcwzg] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-dcwzg for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.171930 3786 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-dcwzg I0428 15:37:30.211598 3786 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s7fh4] pod update took 45.00153ms I0428 15:37:30.211628 3786 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-s7fh4] addLogicalPort annotation time took 45.036541ms I0428 15:37:30.211794 3786 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-67wp8] pod update took 48.622038ms I0428 15:37:30.211805 3786 pod.go:62] [openshift-network-diagnostics/network-check-source-84fd6d9d54-cttkb] pod update took 40.237583ms I0428 15:37:30.211813 3786 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-67wp8] addLogicalPort annotation time took 48.646303ms I0428 15:37:30.211823 3786 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-cttkb] addLogicalPort annotation time took 40.257011ms I0428 15:37:30.212834 3786 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s7fh4] addLogicalPort took 46.316419ms, libovsdb time 860.864µs I0428 15:37:30.213151 3786 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-67wp8] addLogicalPort took 50.114829ms, libovsdb time 1.068502ms I0428 15:37:30.213362 3786 pods.go:271] [openshift-network-diagnostics/network-check-source-84fd6d9d54-cttkb] addLogicalPort took 41.857688ms, libovsdb time 1.243643ms I0428 15:37:30.214137 3786 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-5f578f8577-xnnpr] creating logical port openshift-ingress_router-default-5f578f8577-xnnpr for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.214207 3786 kube.go:256] Updating pod openshift-ingress/router-default-5f578f8577-xnnpr I0428 15:37:30.214639 3786 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-bkn49] pod update took 43.014579ms I0428 15:37:30.214659 3786 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-bkn49] addLogicalPort annotation time took 43.034282ms I0428 15:37:30.215665 3786 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-bkn49] addLogicalPort took 44.092246ms, libovsdb time 774.644µs I0428 15:37:30.218643 3786 pod.go:62] [openshift-ingress-canary/ingress-canary-qnplk] pod update took 47.106792ms I0428 15:37:30.218665 3786 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-qnplk] addLogicalPort annotation time took 47.129596ms I0428 15:37:30.218754 3786 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-tdmlr] pod update took 52.181177ms I0428 15:37:30.218779 3786 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-tdmlr] addLogicalPort annotation time took 52.208978ms I0428 15:37:30.219604 3786 pods.go:271] [openshift-ingress-canary/ingress-canary-qnplk] addLogicalPort took 48.142987ms, libovsdb time 646.735µs I0428 15:37:30.219955 3786 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-tdmlr] addLogicalPort took 53.500026ms, libovsdb time 788.054µs I0428 15:37:30.219986 3786 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-r892r] creating logical port openshift-dns_dns-default-r892r for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:30.220042 3786 kube.go:256] Updating pod openshift-dns/dns-default-r892r I0428 15:37:30.234249 3786 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-dcwzg] pod update took 62.338136ms I0428 15:37:30.234268 3786 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-dcwzg] addLogicalPort annotation time took 62.359104ms I0428 15:37:30.235284 3786 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-dcwzg] addLogicalPort took 63.439994ms, libovsdb time 701.877µs I0428 15:37:30.246514 3786 pod.go:62] [openshift-ingress/router-default-5f578f8577-xnnpr] pod update took 32.328886ms I0428 15:37:30.246518 3786 pod.go:62] [openshift-dns/dns-default-r892r] pod update took 26.49724ms I0428 15:37:30.246534 3786 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-5f578f8577-xnnpr] addLogicalPort annotation time took 32.351186ms I0428 15:37:30.246540 3786 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-r892r] addLogicalPort annotation time took 26.518897ms I0428 15:37:30.247458 3786 pods.go:271] [openshift-ingress/router-default-5f578f8577-xnnpr] addLogicalPort took 33.325865ms, libovsdb time 633.565µs I0428 15:37:30.247706 3786 pods.go:271] [openshift-dns/dns-default-r892r] addLogicalPort took 27.725749ms, libovsdb time 858.538µs I0428 15:37:30.495216 3786 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-2b6k5 a697cb7ec96a422d85cbd0fdf215d5be46200946b65b673a9cb4706c74d84f1a network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-2b6k5 a697cb7ec96a422d85cbd0fdf215d5be46200946b65b673a9cb4706c74d84f1a network default NAD default NAD key ] I0428 15:37:30.521408 3786 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-n6p49 50941447c23e928bec8c099f330e35934a35e39fdb46d21bb51e41b81d5c4efc network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-n6p49 50941447c23e928bec8c099f330e35934a35e39fdb46d21bb51e41b81d5c4efc network default NAD default NAD key ] I0428 15:37:30.533572 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-n6p49, hostIfaceName: 50941447c23e928, network: default, NAD default, SandboxID: "50941447c23e928bec8c099f330e35934a35e39fdb46d21bb51e41b81d5c4efc", PCI device ID: , UID: "9c56931b-99bf-497f-9014-50f24d96dad4", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0428 15:37:30.533575 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-2b6k5, hostIfaceName: a697cb7ec96a422, network: default, NAD default, SandboxID: "a697cb7ec96a422d85cbd0fdf215d5be46200946b65b673a9cb4706c74d84f1a", PCI device ID: , UID: "4b61af46-60e2-4b47-9bd9-367554ea4830", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0428 15:37:30.544176 3786 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-v566m 515aa44279d2187c15894b74abd270f3cdcaab45b341605ae2427d711e9daf21 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-v566m 515aa44279d2187c15894b74abd270f3cdcaab45b341605ae2427d711e9daf21 network default NAD default NAD key ] I0428 15:37:30.558226 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-v566m, hostIfaceName: 515aa44279d2187, network: default, NAD default, SandboxID: "515aa44279d2187c15894b74abd270f3cdcaab45b341605ae2427d711e9daf21", PCI device ID: , UID: "8fe511bb-e41b-46db-a741-dccec2aadc83", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0428 15:37:30.584440 3786 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-tdmlr ac78b590807d84cee2b3f37e091439379c08a93b4fb6eb73e0ca45709efd3f0a network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-tdmlr ac78b590807d84cee2b3f37e091439379c08a93b4fb6eb73e0ca45709efd3f0a network default NAD default NAD key ] I0428 15:37:30.590302 3786 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s7fh4 55436df6c6176188b82323b4dcab100d950df40a43f53f9723783ae4ffca7437 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s7fh4 55436df6c6176188b82323b4dcab100d950df40a43f53f9723783ae4ffca7437 network default NAD default NAD key ] I0428 15:37:30.600143 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-tdmlr, hostIfaceName: ac78b590807d84c, network: default, NAD default, SandboxID: "ac78b590807d84cee2b3f37e091439379c08a93b4fb6eb73e0ca45709efd3f0a", PCI device ID: , UID: "031ff6c9-4c14-459d-a56e-8210dcdf061a", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0428 15:37:30.605900 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-s7fh4, hostIfaceName: 55436df6c617618, network: default, NAD default, SandboxID: "55436df6c6176188b82323b4dcab100d950df40a43f53f9723783ae4ffca7437", PCI device ID: , UID: "a3c9d636-d3df-4f4b-80c2-e98c1c69f820", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0428 15:37:30.607300 3786 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-2b6k5 a697cb7ec96a422d85cbd0fdf215d5be46200946b65b673a9cb4706c74d84f1a network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-2b6k5 a697cb7ec96a422d85cbd0fdf215d5be46200946b65b673a9cb4706c74d84f1a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:59:4b:80:38:bd\",\"name\":\"a697cb7ec96a422\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d24e6118-0633-45e7-9cd5-0bd1cc28ade4\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:30.623305 3786 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-n6p49 50941447c23e928bec8c099f330e35934a35e39fdb46d21bb51e41b81d5c4efc network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-n6p49 50941447c23e928bec8c099f330e35934a35e39fdb46d21bb51e41b81d5c4efc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:ae:7f:38:cc:54\",\"name\":\"50941447c23e928\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b50182eb-6a68-43c1-b474-cc4b19f105ff\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:30.623640 3786 cni.go:452] [openshift-ingress-canary/ingress-canary-qnplk fe31f7b382a4aeaa3510ebc72361ef77c2184f34a6078068cbab0c3bcd8889ec network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-qnplk fe31f7b382a4aeaa3510ebc72361ef77c2184f34a6078068cbab0c3bcd8889ec network default NAD default NAD key ] I0428 15:37:30.630793 3786 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-v566m 515aa44279d2187c15894b74abd270f3cdcaab45b341605ae2427d711e9daf21 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-v566m 515aa44279d2187c15894b74abd270f3cdcaab45b341605ae2427d711e9daf21 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:f0:80:27:f3:c5\",\"name\":\"515aa44279d2187\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0c6d2785-8f58-4404-b762-0562e7ec4adb\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:30.636740 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-qnplk, hostIfaceName: fe31f7b382a4aea, network: default, NAD default, SandboxID: "fe31f7b382a4aeaa3510ebc72361ef77c2184f34a6078068cbab0c3bcd8889ec", PCI device ID: , UID: "ec6aa4b9-fff1-4a7c-b074-115a15e45f76", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0428 15:37:30.639044 3786 cni.go:452] [openshift-network-diagnostics/network-check-source-84fd6d9d54-cttkb 7068e1fad3d2feab4a58482dfe0e5200c244325f4b8526206d4359c4b55baf9a network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-cttkb 7068e1fad3d2feab4a58482dfe0e5200c244325f4b8526206d4359c4b55baf9a network default NAD default NAD key ] I0428 15:37:30.650855 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-84fd6d9d54-cttkb, hostIfaceName: 7068e1fad3d2fea, network: default, NAD default, SandboxID: "7068e1fad3d2feab4a58482dfe0e5200c244325f4b8526206d4359c4b55baf9a", PCI device ID: , UID: "af3828dc-da90-4885-bd4a-7fc29dcd1daa", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0428 15:37:30.656980 3786 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-bkn49 ac0de941596dbf7d5b0ec7111022226063d0f164d11bf231570630cdd5a8127e network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-bkn49 ac0de941596dbf7d5b0ec7111022226063d0f164d11bf231570630cdd5a8127e network default NAD default NAD key ] I0428 15:37:30.669924 3786 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s7fh4 55436df6c6176188b82323b4dcab100d950df40a43f53f9723783ae4ffca7437 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-s7fh4 55436df6c6176188b82323b4dcab100d950df40a43f53f9723783ae4ffca7437 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:de:49:28:10:4a\",\"name\":\"55436df6c617618\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/59cbe670-9c26-4738-84e7-76853df7f0c7\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:30.671335 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-bkn49, hostIfaceName: ac0de941596dbf7, network: default, NAD default, SandboxID: "ac0de941596dbf7d5b0ec7111022226063d0f164d11bf231570630cdd5a8127e", PCI device ID: , UID: "f3c4e1cc-fb44-4330-9b13-a25e60085d8f", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0428 15:37:30.676311 3786 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-tdmlr ac78b590807d84cee2b3f37e091439379c08a93b4fb6eb73e0ca45709efd3f0a network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-tdmlr ac78b590807d84cee2b3f37e091439379c08a93b4fb6eb73e0ca45709efd3f0a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:fc:5c:1b:0f:5a\",\"name\":\"ac78b590807d84c\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fdffe468-8f75-457c-bfb0-2da2b7222da8\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:30.695812 3786 cni.go:473] [openshift-ingress-canary/ingress-canary-qnplk fe31f7b382a4aeaa3510ebc72361ef77c2184f34a6078068cbab0c3bcd8889ec network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-qnplk fe31f7b382a4aeaa3510ebc72361ef77c2184f34a6078068cbab0c3bcd8889ec network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:2d:07:d0:86:d5\",\"name\":\"fe31f7b382a4aea\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/be17bbfa-e21d-4f22-8120-4b86120e4dfb\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:30.706392 3786 cni.go:473] [openshift-network-diagnostics/network-check-source-84fd6d9d54-cttkb 7068e1fad3d2feab4a58482dfe0e5200c244325f4b8526206d4359c4b55baf9a network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-cttkb 7068e1fad3d2feab4a58482dfe0e5200c244325f4b8526206d4359c4b55baf9a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:5b:e7:8e:59:ea\",\"name\":\"7068e1fad3d2fea\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2861fd00-7ba7-4d65-accf-f39ed8c73546\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:30.723651 3786 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-bkn49 ac0de941596dbf7d5b0ec7111022226063d0f164d11bf231570630cdd5a8127e network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-bkn49 ac0de941596dbf7d5b0ec7111022226063d0f164d11bf231570630cdd5a8127e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:b6:83:b1:2a:59\",\"name\":\"ac0de941596dbf7\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/403c046d-dee0-4392-90c1-ead7ac028053\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:32.307586 3786 default_network_controller.go:1025] Node "ip-10-0-1-106.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:37.455450 3786 namespace.go:98] [openshift-service-ca] adding namespace I0428 15:37:37.456328 3786 namespace.go:102] [openshift-service-ca] adding namespace took 855.509µs I0428 15:37:37.476383 3786 namespace.go:142] [openshift-service-ca] updating namespace I0428 15:37:39.191958 3786 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-mrsxs] creating logical port openshift-service-ca_service-ca-d9cb48497-mrsxs for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:39.192120 3786 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-mrsxs I0428 15:37:39.214910 3786 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-mrsxs] pod update took 22.835681ms I0428 15:37:39.214932 3786 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-mrsxs] addLogicalPort annotation time took 22.862185ms I0428 15:37:39.216152 3786 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-mrsxs] addLogicalPort took 24.210723ms, libovsdb time 854.343µs I0428 15:37:39.753470 3786 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-mrsxs 63f91c947ee231856b927081f6f3fe6c2f628149729f81729e0466bc50823999 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-mrsxs 63f91c947ee231856b927081f6f3fe6c2f628149729f81729e0466bc50823999 network default NAD default NAD key ] I0428 15:37:39.764785 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-mrsxs, hostIfaceName: 63f91c947ee2318, network: default, NAD default, SandboxID: "63f91c947ee231856b927081f6f3fe6c2f628149729f81729e0466bc50823999", PCI device ID: , UID: "e32c7dd4-6c7f-4b4f-96d9-f992ce631a02", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0428 15:37:39.819289 3786 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-mrsxs 63f91c947ee231856b927081f6f3fe6c2f628149729f81729e0466bc50823999 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-mrsxs 63f91c947ee231856b927081f6f3fe6c2f628149729f81729e0466bc50823999 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:e5:c7:c9:3f:50\",\"name\":\"63f91c947ee2318\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/17d7a70d-fa33-4b53-b446-ebf81d4cb8b5\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:40.182965 3786 default_network_controller.go:1025] Node "ip-10-0-0-9.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:40.204781 3786 default_network_controller.go:1025] Node "ip-10-0-1-106.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0428 15:37:40.978411 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0428 15:37:41.005153 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0428 15:37:41.030599 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:37:41.030604 3786 egressfirewall.go:723] Syncing node "ip-10-0-1-106.ec2.internal" for egress firewall I0428 15:37:41.030620 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 28.186µs I0428 15:37:41.030632 3786 default_network_controller.go:1025] Node "ip-10-0-1-106.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:41.898139 3786 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0428 15:37:41.899169 3786 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 999.174µs I0428 15:37:41.951296 3786 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0428 15:37:43.008485 3786 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-2jwxd] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-2jwxd for pod on switch ip-10-0-2-7.ec2.internal I0428 15:37:43.008612 3786 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-2jwxd I0428 15:37:43.036159 3786 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-2jwxd] pod update took 27.583464ms I0428 15:37:43.036189 3786 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-2jwxd] addLogicalPort annotation time took 27.618148ms I0428 15:37:43.037466 3786 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-2jwxd] addLogicalPort took 28.998686ms, libovsdb time 895.986µs I0428 15:37:43.366649 3786 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-2jwxd 4bb6a6021a3dced75d3a65daa4b2554c77e5381f41b8c303e96017347712cf95 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-2jwxd 4bb6a6021a3dced75d3a65daa4b2554c77e5381f41b8c303e96017347712cf95 network default NAD default NAD key ] I0428 15:37:43.377650 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-2jwxd, hostIfaceName: 4bb6a6021a3dced, network: default, NAD default, SandboxID: "4bb6a6021a3dced75d3a65daa4b2554c77e5381f41b8c303e96017347712cf95", PCI device ID: , UID: "f1c31ebd-13ae-431c-89cc-c00ab698c77b", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0428 15:37:43.430347 3786 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-2jwxd 4bb6a6021a3dced75d3a65daa4b2554c77e5381f41b8c303e96017347712cf95 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-2jwxd 4bb6a6021a3dced75d3a65daa4b2554c77e5381f41b8c303e96017347712cf95 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:04:eb:4b:a8:4d\",\"name\":\"4bb6a6021a3dced\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c705123a-2a54-4307-ba58-19c564a76ea8\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:37:44.803046 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:37:45.241270 3786 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-6669988f88-tj54w IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-6669988f88-tj54w: no pod IPs found W0428 15:37:45.272511 3786 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-7fc7c49d95-ctrtf IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-7fc7c49d95-ctrtf: no pod IPs found I0428 15:37:45.732165 3786 default_network_controller.go:1025] Node "ip-10-0-0-9.ec2.internal" in remote zone "ip-10-0-0-9.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:45.732253 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-9.ec2.internal in network "default" I0428 15:37:45.732297 3786 node_tracker.go:235] Node ip-10-0-0-9.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-9.ec2.internal" I0428 15:37:45.732316 3786 node_tracker.go:169] Node ip-10-0-0-9.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.732330 3786 services_controller.go:574] Full service sync requested for network=default W0428 15:37:45.732790 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:37:45.732802 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:45.772596 3786 default_network_controller.go:1025] Node "ip-10-0-0-9.ec2.internal" in remote zone "ip-10-0-0-9.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:45.860061 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:45.860125 3786 default_network_controller.go:1025] Node "ip-10-0-1-106.ec2.internal" in remote zone "ip-10-0-1-106.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:45.860124 3786 node_tracker.go:235] Node ip-10-0-1-106.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-106.ec2.internal" I0428 15:37:45.860151 3786 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.860169 3786 services_controller.go:574] Full service sync requested for network=default W0428 15:37:45.860515 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:37:45.860544 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:45.876993 3786 default_network_controller.go:1025] Node "ip-10-0-0-9.ec2.internal" in remote zone "ip-10-0-0-9.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:45.877052 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-9.ec2.internal in network "default" I0428 15:37:45.877133 3786 node_tracker.go:169] Node ip-10-0-0-9.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.877151 3786 services_controller.go:574] Full service sync requested for network=default W0428 15:37:45.877227 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:37:45.877292 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:45.877965 3786 zone_ic_handler.go:195] Time taken to create transit switch: 252.044µs I0428 15:37:45.877979 3786 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-9.ec2.internal for the network default I0428 15:37:45.878980 3786 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-9.ec2.internal" on network "default" took: 1.278841ms I0428 15:37:45.894630 3786 default_network_controller.go:1025] Node "ip-10-0-1-106.ec2.internal" in remote zone "ip-10-0-1-106.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:46.017973 3786 default_network_controller.go:1025] Node "ip-10-0-1-106.ec2.internal" in remote zone "ip-10-0-1-106.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:46.018068 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:46.018143 3786 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:46.018159 3786 services_controller.go:574] Full service sync requested for network=default W0428 15:37:46.018253 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0428 15:37:46.018252 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0428 15:37:46.019111 3786 zone_ic_handler.go:195] Time taken to create transit switch: 320.069µs I0428 15:37:46.019130 3786 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-106.ec2.internal for the network default I0428 15:37:46.020247 3786 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-106.ec2.internal" on network "default" took: 1.469822ms I0428 15:37:46.148712 3786 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-fwx5l f8e35a39fc2f739891fe83fa8b327dad6e701b7e801776db4c1d07a2229ee7ab network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-fwx5l f8e35a39fc2f739891fe83fa8b327dad6e701b7e801776db4c1d07a2229ee7ab network default NAD default NAD key ] I0428 15:37:46.158958 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-fwx5l, hostIfaceName: f8e35a39fc2f739, network: default, NAD default, SandboxID: "f8e35a39fc2f739891fe83fa8b327dad6e701b7e801776db4c1d07a2229ee7ab", PCI device ID: , UID: "1e5ebc8a-2146-4d73-9ec1-10fb152a7bdf", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0428 15:37:46.211052 3786 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-fwx5l f8e35a39fc2f739891fe83fa8b327dad6e701b7e801776db4c1d07a2229ee7ab network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-fwx5l f8e35a39fc2f739891fe83fa8b327dad6e701b7e801776db4c1d07a2229ee7ab network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:dd:e3:c1:02:96\",\"name\":\"f8e35a39fc2f739\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c2490f51-3d76-43c3-8a06-8584a98e8587\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:46.290171 3786 cni.go:452] [openshift-dns/dns-default-r892r 8c91216a8a180886a141402c95efb9560723c239334b63f7d0cad2d7dc98911a network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-r892r 8c91216a8a180886a141402c95efb9560723c239334b63f7d0cad2d7dc98911a network default NAD default NAD key ] I0428 15:37:46.300583 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-r892r, hostIfaceName: 8c91216a8a18088, network: default, NAD default, SandboxID: "8c91216a8a180886a141402c95efb9560723c239334b63f7d0cad2d7dc98911a", PCI device ID: , UID: "2bb3a9a7-d775-4410-993a-970ce1abcbf2", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0428 15:37:46.355810 3786 cni.go:473] [openshift-dns/dns-default-r892r 8c91216a8a180886a141402c95efb9560723c239334b63f7d0cad2d7dc98911a network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-r892r 8c91216a8a180886a141402c95efb9560723c239334b63f7d0cad2d7dc98911a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:d5:2a:35:a2:ae\",\"name\":\"8c91216a8a18088\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5d1ce4ad-828e-45f3-809d-a72010e18ebd\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:46.362060 3786 cni.go:452] [openshift-image-registry/image-registry-7fc7c49d95-dgxkp f832256019b119720e7815b9c549ed4bb19c53dc54222204c349a31204622fa6 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7fc7c49d95-dgxkp f832256019b119720e7815b9c549ed4bb19c53dc54222204c349a31204622fa6 network default NAD default NAD key ] I0428 15:37:46.371490 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7fc7c49d95-dgxkp, hostIfaceName: f832256019b1197, network: default, NAD default, SandboxID: "f832256019b119720e7815b9c549ed4bb19c53dc54222204c349a31204622fa6", PCI device ID: , UID: "eec56deb-b8a1-4b87-842e-cc09f261fac0", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0428 15:37:46.426213 3786 cni.go:473] [openshift-image-registry/image-registry-7fc7c49d95-dgxkp f832256019b119720e7815b9c549ed4bb19c53dc54222204c349a31204622fa6 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7fc7c49d95-dgxkp f832256019b119720e7815b9c549ed4bb19c53dc54222204c349a31204622fa6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:b6:7f:7f:46:08\",\"name\":\"f832256019b1197\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ac9ccc13-d413-49aa-ba8a-55a78ed4118f\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:37:46.960474 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:37:46.960492 3786 egressfirewall.go:723] Syncing node "ip-10-0-0-9.ec2.internal" for egress firewall I0428 15:37:46.960502 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 34.124µs I0428 15:37:46.960476 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-9.ec2.internal in network "default" I0428 15:37:46.960554 3786 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-9.ec2.internal", IP: 10.0.0.9 to PMTUD blocking rules I0428 15:37:46.960569 3786 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-9.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:37:46.960598 3786 node_tracker.go:169] Node ip-10-0-0-9.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:46.960620 3786 services_controller.go:574] Full service sync requested for network=default W0428 15:37:46.961072 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:37:46.961083 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:46.966644 3786 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-9.ec2.internal. Updating no re-route policies I0428 15:37:46.980605 3786 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-9.ec2.internal", IP: 10.0.0.9 to PMTUD blocking rules I0428 15:37:46.980628 3786 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-9.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:37:48.150992 3786 egressfirewall.go:723] Syncing node "ip-10-0-1-106.ec2.internal" for egress firewall I0428 15:37:48.151001 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:37:48.151034 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 39.804µs I0428 15:37:48.151132 3786 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:48.151184 3786 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:48.151200 3786 services_controller.go:574] Full service sync requested for network=default I0428 15:37:48.151196 3786 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-106.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:37:48.151210 3786 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-106.ec2.internal", IP: 10.0.1.106 to PMTUD blocking rules W0428 15:37:48.151719 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:37:48.151743 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:48.157460 3786 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-106.ec2.internal. Updating no re-route policies I0428 15:37:48.175717 3786 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-106.ec2.internal", IP: 10.0.1.106 to PMTUD blocking rules I0428 15:37:48.175739 3786 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-106.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:38:01.726342 3786 cni.go:452] [openshift-network-diagnostics/network-check-target-89vsl 3409bde08e8044efed904f78be5e955f267ccd2b4ea505d1c8c44fd0a62ea118 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-89vsl 3409bde08e8044efed904f78be5e955f267ccd2b4ea505d1c8c44fd0a62ea118 network default NAD default NAD key ] I0428 15:38:01.728780 3786 cni.go:452] [openshift-multus/network-metrics-daemon-tvj87 ae2646d50e5874c1800ab2fd56ceacbae63b6499c3cb7cd24ba96a51deaa6410 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-tvj87 ae2646d50e5874c1800ab2fd56ceacbae63b6499c3cb7cd24ba96a51deaa6410 network default NAD default NAD key ] I0428 15:38:01.737479 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-89vsl, hostIfaceName: 3409bde08e8044e, network: default, NAD default, SandboxID: "3409bde08e8044efed904f78be5e955f267ccd2b4ea505d1c8c44fd0a62ea118", PCI device ID: , UID: "7829b2b1-14eb-468f-95c6-1c8de034aa5b", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0428 15:38:01.738610 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-tvj87, hostIfaceName: ae2646d50e5874c, network: default, NAD default, SandboxID: "ae2646d50e5874c1800ab2fd56ceacbae63b6499c3cb7cd24ba96a51deaa6410", PCI device ID: , UID: "32cbdc7e-4dae-4843-89b7-5dcd232758f0", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0428 15:38:01.791850 3786 cni.go:473] [openshift-multus/network-metrics-daemon-tvj87 ae2646d50e5874c1800ab2fd56ceacbae63b6499c3cb7cd24ba96a51deaa6410 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-tvj87 ae2646d50e5874c1800ab2fd56ceacbae63b6499c3cb7cd24ba96a51deaa6410 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:b3:61:67:a5:46\",\"name\":\"ae2646d50e5874c\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9b1dc183-3d77-47bd-94ed-9f03bd26bdb3\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:38:01.806046 3786 cni.go:473] [openshift-network-diagnostics/network-check-target-89vsl 3409bde08e8044efed904f78be5e955f267ccd2b4ea505d1c8c44fd0a62ea118 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-89vsl 3409bde08e8044efed904f78be5e955f267ccd2b4ea505d1c8c44fd0a62ea118 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:cf:33:3d:65:f6\",\"name\":\"3409bde08e8044e\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2bc138e1-c131-494a-9ede-d236b1cae7d9\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:38:02.293649 3786 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-9vnss 7f5d371eb26541e86de29f9e2eae3a8e96e5e6f9666c7afb4193b11cba815560 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-9vnss 7f5d371eb26541e86de29f9e2eae3a8e96e5e6f9666c7afb4193b11cba815560 network default NAD default NAD key ] I0428 15:38:02.306776 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-9vnss, hostIfaceName: 7f5d371eb26541e, network: default, NAD default, SandboxID: "7f5d371eb26541e86de29f9e2eae3a8e96e5e6f9666c7afb4193b11cba815560", PCI device ID: , UID: "a230dcf7-7306-4ffa-be5b-aea8802b0f78", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0428 15:38:02.329635 3786 cni.go:452] [openshift-ingress/router-default-6d4c95c6c5-nsvcp f6ad3d12200ff4102770efcf278e39bdb9903696aa8a662f250ac54ec455776f network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-6d4c95c6c5-nsvcp f6ad3d12200ff4102770efcf278e39bdb9903696aa8a662f250ac54ec455776f network default NAD default NAD key ] I0428 15:38:02.343502 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-6d4c95c6c5-nsvcp, hostIfaceName: f6ad3d12200ff41, network: default, NAD default, SandboxID: "f6ad3d12200ff4102770efcf278e39bdb9903696aa8a662f250ac54ec455776f", PCI device ID: , UID: "21bdb6ca-0f71-4fa6-a1ca-b3d966813115", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0428 15:38:02.360132 3786 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-67wp8 9e366f8b15254486b38a31efeb6aa66043d25d4c007cc56966c86cbe213d6110 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-67wp8 9e366f8b15254486b38a31efeb6aa66043d25d4c007cc56966c86cbe213d6110 network default NAD default NAD key ] I0428 15:38:02.361128 3786 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-9vnss 7f5d371eb26541e86de29f9e2eae3a8e96e5e6f9666c7afb4193b11cba815560 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-9vnss 7f5d371eb26541e86de29f9e2eae3a8e96e5e6f9666c7afb4193b11cba815560 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:cb:13:c6:19:c2\",\"name\":\"7f5d371eb26541e\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b0ff5a69-3006-4f04-8224-63e57087ab43\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:38:02.372554 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-67wp8, hostIfaceName: 9e366f8b1525448, network: default, NAD default, SandboxID: "9e366f8b15254486b38a31efeb6aa66043d25d4c007cc56966c86cbe213d6110", PCI device ID: , UID: "6e91efb7-0a47-4239-8385-825de3f3abbf", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0428 15:38:02.399062 3786 cni.go:473] [openshift-ingress/router-default-6d4c95c6c5-nsvcp f6ad3d12200ff4102770efcf278e39bdb9903696aa8a662f250ac54ec455776f network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-6d4c95c6c5-nsvcp f6ad3d12200ff4102770efcf278e39bdb9903696aa8a662f250ac54ec455776f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:7f:d5:0a:1c:20\",\"name\":\"f6ad3d12200ff41\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe90ed74-273a-431d-ac00-097a1984070f\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:38:02.426272 3786 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-67wp8 9e366f8b15254486b38a31efeb6aa66043d25d4c007cc56966c86cbe213d6110 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-67wp8 9e366f8b15254486b38a31efeb6aa66043d25d4c007cc56966c86cbe213d6110 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:a1:34:d5:66:d1\",\"name\":\"9e366f8b1525448\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3ac853ee-fbf0-4972-9edd-77f813fab509\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:38:02.437014 3786 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-dcwzg 919089ed58771b8dade50e10497f1514473d07bd14f550f8d5f9bed0ceaddced network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-dcwzg 919089ed58771b8dade50e10497f1514473d07bd14f550f8d5f9bed0ceaddced network default NAD default NAD key ] I0428 15:38:02.454393 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-dcwzg, hostIfaceName: 919089ed58771b8, network: default, NAD default, SandboxID: "919089ed58771b8dade50e10497f1514473d07bd14f550f8d5f9bed0ceaddced", PCI device ID: , UID: "eff770b9-de51-4768-bc61-4e3253a28f15", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0428 15:38:02.467774 3786 cni.go:452] [openshift-ingress/router-default-5f578f8577-xnnpr c3986e0fd8518323fce62924225b4b9bd82709a1702cdbc38ca257a2e6f43d66 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-5f578f8577-xnnpr c3986e0fd8518323fce62924225b4b9bd82709a1702cdbc38ca257a2e6f43d66 network default NAD default NAD key ] I0428 15:38:02.478190 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-5f578f8577-xnnpr, hostIfaceName: c3986e0fd851832, network: default, NAD default, SandboxID: "c3986e0fd8518323fce62924225b4b9bd82709a1702cdbc38ca257a2e6f43d66", PCI device ID: , UID: "40b5c2b3-3221-4d01-97e8-b0d854057f52", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0428 15:38:02.518310 3786 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-dcwzg 919089ed58771b8dade50e10497f1514473d07bd14f550f8d5f9bed0ceaddced network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-dcwzg 919089ed58771b8dade50e10497f1514473d07bd14f550f8d5f9bed0ceaddced network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:02:5f:29:ed:82\",\"name\":\"919089ed58771b8\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/23ec7ccb-d12a-44bb-8537-b1752accc2ca\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:38:02.538777 3786 cni.go:473] [openshift-ingress/router-default-5f578f8577-xnnpr c3986e0fd8518323fce62924225b4b9bd82709a1702cdbc38ca257a2e6f43d66 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-5f578f8577-xnnpr c3986e0fd8518323fce62924225b4b9bd82709a1702cdbc38ca257a2e6f43d66 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:2e:8f:51:a3:3d\",\"name\":\"c3986e0fd851832\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/279386c7-6347-432d-bf3e-d816bc41dd4e\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:38:03.225194 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0428 15:38:03.236524 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:03.236548 3786 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:03.236555 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:03.236565 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:03.236584 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:03.253078 3786 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:38:03.255762 3786 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:03.347636 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0428 15:38:03.357707 3786 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-59x2d] creating logical port openshift-insights_insights-runtime-extractor-59x2d for pod on switch ip-10-0-2-7.ec2.internal I0428 15:38:03.357879 3786 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-59x2d I0428 15:38:03.438222 3786 pod.go:62] [openshift-insights/insights-runtime-extractor-59x2d] pod update took 80.38879ms I0428 15:38:03.438247 3786 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-59x2d] addLogicalPort annotation time took 80.422384ms I0428 15:38:03.439493 3786 pods.go:271] [openshift-insights/insights-runtime-extractor-59x2d] addLogicalPort took 81.810549ms, libovsdb time 910.987µs I0428 15:38:03.441909 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:38:03.441982 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 77.103µs W0428 15:38:03.492664 3786 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-796c59fd79-92mqs IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-796c59fd79-92mqs: no pod IPs found W0428 15:38:03.511441 3786 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:03.823603 3786 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-g7chq] creating logical port openshift-console_downloads-5c54f4944f-g7chq for pod on switch ip-10-0-2-7.ec2.internal I0428 15:38:03.823705 3786 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-g7chq I0428 15:38:03.916432 3786 pod.go:62] [openshift-console/downloads-5c54f4944f-g7chq] pod update took 92.752936ms I0428 15:38:03.916462 3786 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-g7chq] addLogicalPort annotation time took 92.787335ms I0428 15:38:03.917845 3786 pods.go:271] [openshift-console/downloads-5c54f4944f-g7chq] addLogicalPort took 94.262983ms, libovsdb time 1.004027ms I0428 15:38:03.991545 3786 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:38:03.991577 3786 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 41.391µs W0428 15:38:04.203422 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:04.203445 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:04.203452 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:04.203594 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:04.203603 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0428 15:38:04.352155 3786 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 15:38:04.457978 3786 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 15:38:04.493971 3786 cni.go:452] [openshift-console/downloads-5c54f4944f-g7chq dd68982d26d3b5a6c47e52a12c7d07f4a82f20767a5eda91450d7ef154b03ca1 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-g7chq dd68982d26d3b5a6c47e52a12c7d07f4a82f20767a5eda91450d7ef154b03ca1 network default NAD default NAD key ] I0428 15:38:04.503855 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-g7chq, hostIfaceName: dd68982d26d3b5a, network: default, NAD default, SandboxID: "dd68982d26d3b5a6c47e52a12c7d07f4a82f20767a5eda91450d7ef154b03ca1", PCI device ID: , UID: "fdf61f33-07ae-4959-a2cb-5dd9a80e0c98", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0428 15:38:04.556867 3786 cni.go:473] [openshift-console/downloads-5c54f4944f-g7chq dd68982d26d3b5a6c47e52a12c7d07f4a82f20767a5eda91450d7ef154b03ca1 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-g7chq dd68982d26d3b5a6c47e52a12c7d07f4a82f20767a5eda91450d7ef154b03ca1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:04:41:0b:2c:68\",\"name\":\"dd68982d26d3b5a\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b497c298-bee1-4650-b170-b8bd4c427942\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:38:05.208602 3786 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:05.220069 3786 cni.go:452] [openshift-insights/insights-runtime-extractor-59x2d 16eaedecdb2d4c5b48dfa28e098601e5fa4ba8e6c7fe72c40107fc7625c24ae0 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-59x2d 16eaedecdb2d4c5b48dfa28e098601e5fa4ba8e6c7fe72c40107fc7625c24ae0 network default NAD default NAD key ] I0428 15:38:05.229490 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-59x2d, hostIfaceName: 16eaedecdb2d4c5, network: default, NAD default, SandboxID: "16eaedecdb2d4c5b48dfa28e098601e5fa4ba8e6c7fe72c40107fc7625c24ae0", PCI device ID: , UID: "9a76d231-c853-48fb-86dd-b4cb12ab7574", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] W0428 15:38:05.243078 3786 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:05.283878 3786 cni.go:473] [openshift-insights/insights-runtime-extractor-59x2d 16eaedecdb2d4c5b48dfa28e098601e5fa4ba8e6c7fe72c40107fc7625c24ae0 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-59x2d 16eaedecdb2d4c5b48dfa28e098601e5fa4ba8e6c7fe72c40107fc7625c24ae0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:fe:72:72:da:85\",\"name\":\"16eaedecdb2d4c5\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8615b3f2-e1c3-48fb-b2b3-8335974266f3\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:38:09.266549 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0428 15:38:13.478915 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:15.540245 3786 base_network_controller_pods.go:487] [default/openshift-console/console-56b489d6fc-b8dxl] creating logical port openshift-console_console-56b489d6fc-b8dxl for pod on switch ip-10-0-2-7.ec2.internal I0428 15:38:15.540356 3786 kube.go:256] Updating pod openshift-console/console-56b489d6fc-b8dxl I0428 15:38:15.568361 3786 pod.go:62] [openshift-console/console-56b489d6fc-b8dxl] pod update took 28.038592ms I0428 15:38:15.568385 3786 base_network_controller_pods.go:951] [default/openshift-console/console-56b489d6fc-b8dxl] addLogicalPort annotation time took 28.068218ms I0428 15:38:15.569733 3786 pods.go:271] [openshift-console/console-56b489d6fc-b8dxl] addLogicalPort took 29.507839ms, libovsdb time 948.664µs I0428 15:38:15.904200 3786 cni.go:452] [openshift-console/console-56b489d6fc-b8dxl e45e8a39c461fc12e83a8a9d85724cd0e752c89fbad542cc1a53cedd5c34f39f network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-56b489d6fc-b8dxl e45e8a39c461fc12e83a8a9d85724cd0e752c89fbad542cc1a53cedd5c34f39f network default NAD default NAD key ] I0428 15:38:15.914438 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-56b489d6fc-b8dxl, hostIfaceName: e45e8a39c461fc1, network: default, NAD default, SandboxID: "e45e8a39c461fc12e83a8a9d85724cd0e752c89fbad542cc1a53cedd5c34f39f", PCI device ID: , UID: "91310f57-d6c0-4285-a0af-d887a6f04610", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0428 15:38:15.969871 3786 cni.go:473] [openshift-console/console-56b489d6fc-b8dxl e45e8a39c461fc12e83a8a9d85724cd0e752c89fbad542cc1a53cedd5c34f39f network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-56b489d6fc-b8dxl e45e8a39c461fc12e83a8a9d85724cd0e752c89fbad542cc1a53cedd5c34f39f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:31:07:f0:be:38\",\"name\":\"e45e8a39c461fc1\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5f67a6f6-64f7-4ce8-a383-b1e68c1b26e0\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:38:16.280985 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 15:38:16.292482 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 15:38:22.341728 3786 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:22.341748 3786 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:22.341755 3786 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:24.320690 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0428 15:38:25.775131 3786 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-56b489d6fc-lccqq IP address from the namespace address-set, err: pod openshift-console/console-56b489d6fc-lccqq: no pod IPs found W0428 15:38:26.174134 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0428 15:38:28.356436 3786 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:28.356461 3786 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:28.356469 3786 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 15:38:28.611777 3786 cni.go:452] [openshift-image-registry/image-registry-7fc7c49d95-dgxkp f832256019b119720e7815b9c549ed4bb19c53dc54222204c349a31204622fa6 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-7fc7c49d95-dgxkp f832256019b119720e7815b9c549ed4bb19c53dc54222204c349a31204622fa6 network default NAD default NAD key ] I0428 15:38:28.700342 3786 cni.go:473] [openshift-image-registry/image-registry-7fc7c49d95-dgxkp f832256019b119720e7815b9c549ed4bb19c53dc54222204c349a31204622fa6 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-7fc7c49d95-dgxkp f832256019b119720e7815b9c549ed4bb19c53dc54222204c349a31204622fa6 network default NAD default NAD key default], result "{}", err I0428 15:38:28.765777 3786 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:28.765818 3786 pods.go:176] Deleting pod: openshift-image-registry/image-registry-7fc7c49d95-dgxkp I0428 15:38:28.767305 3786 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-7fc7c49d95-dgxkp, ips: 10.128.0.5 I0428 15:38:28.767345 3786 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:28.778008 3786 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-7d9d855f7f-q9p2c] creating logical port openshift-image-registry_image-registry-7d9d855f7f-q9p2c for pod on switch ip-10-0-2-7.ec2.internal I0428 15:38:28.778172 3786 kube.go:256] Updating pod openshift-image-registry/image-registry-7d9d855f7f-q9p2c I0428 15:38:28.798560 3786 pod.go:62] [openshift-image-registry/image-registry-7d9d855f7f-q9p2c] pod update took 20.413695ms I0428 15:38:28.798588 3786 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-7d9d855f7f-q9p2c] addLogicalPort annotation time took 20.44866ms I0428 15:38:28.799945 3786 pods.go:271] [openshift-image-registry/image-registry-7d9d855f7f-q9p2c] addLogicalPort took 21.956727ms, libovsdb time 1.025725ms I0428 15:38:29.119877 3786 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:29.119905 3786 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:29.132577 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7fc7c49d95-dgxkp I0428 15:38:29.132614 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7fc7c49d95-dgxkp I0428 15:38:29.143765 3786 cni.go:452] [openshift-image-registry/image-registry-7d9d855f7f-q9p2c 382915a6beae4c8d054657be536d70f22150b71c01994f6eda778d67b7b5b713 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-7d9d855f7f-q9p2c 382915a6beae4c8d054657be536d70f22150b71c01994f6eda778d67b7b5b713 network default NAD default NAD key ] I0428 15:38:29.154457 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-7d9d855f7f-q9p2c, hostIfaceName: 382915a6beae4c8, network: default, NAD default, SandboxID: "382915a6beae4c8d054657be536d70f22150b71c01994f6eda778d67b7b5b713", PCI device ID: , UID: "b9c19eb2-6699-451f-8e99-8a91bfef926b", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0428 15:38:29.213827 3786 cni.go:473] [openshift-image-registry/image-registry-7d9d855f7f-q9p2c 382915a6beae4c8d054657be536d70f22150b71c01994f6eda778d67b7b5b713 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-7d9d855f7f-q9p2c 382915a6beae4c8d054657be536d70f22150b71c01994f6eda778d67b7b5b713 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:39:08:4f:a5:51\",\"name\":\"382915a6beae4c8\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a1b9e28c-9363-4f44-a87f-be913f97bebe\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:38:29.343073 3786 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.343115 3786 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.440366 3786 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:38:29.514620 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:38:29.605288 3786 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.605313 3786 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.688060 3786 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-69f694b5b7-j8xh7 IP address from the namespace address-set, err: pod openshift-console/console-69f694b5b7-j8xh7: no pod IPs found W0428 15:38:29.695184 3786 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-69f694b5b7-9ddmp IP address from the namespace address-set, err: pod openshift-console/console-69f694b5b7-9ddmp: no pod IPs found W0428 15:38:30.521040 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0428 15:38:31.423592 3786 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.424194 3786 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.424231 3786 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.424243 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0428 15:38:31.424256 3786 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.424261 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... W0428 15:38:32.350987 3786 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 15:38:32.450034 3786 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:32.450056 3786 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:32.450071 3786 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:32.450080 3786 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:38:32.989475 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 15:38:33.198047 3786 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 15:38:34.773069 3786 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0428 15:38:34.788111 3786 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0428 15:38:41.509495 3786 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-n8rgx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:41.510018 3786 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-n8rgx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:42.411391 3786 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-n8rgx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:42.411411 3786 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-n8rgx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:42.414645 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-957cdcb8f-n8rgx I0428 15:38:42.414671 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-957cdcb8f-n8rgx W0428 15:38:46.194487 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 15:38:46.194593 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 15:38:46.194651 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 15:38:47.192192 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 15:38:51.204567 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:51.238740 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0428 15:38:51.238793 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0428 15:38:51.238741 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0428 15:38:51.648331 3786 obj_retry.go:498] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:51.649386 3786 obj_retry.go:498] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0428 15:38:52.237815 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics I0428 15:38:52.269969 3786 cni.go:452] [openshift-console/console-56b489d6fc-b8dxl e45e8a39c461fc12e83a8a9d85724cd0e752c89fbad542cc1a53cedd5c34f39f network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-56b489d6fc-b8dxl e45e8a39c461fc12e83a8a9d85724cd0e752c89fbad542cc1a53cedd5c34f39f network default NAD default NAD key ] I0428 15:38:52.355318 3786 cni.go:473] [openshift-console/console-56b489d6fc-b8dxl e45e8a39c461fc12e83a8a9d85724cd0e752c89fbad542cc1a53cedd5c34f39f network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-56b489d6fc-b8dxl e45e8a39c461fc12e83a8a9d85724cd0e752c89fbad542cc1a53cedd5c34f39f network default NAD default NAD key default], result "{}", err I0428 15:38:52.428750 3786 obj_retry.go:498] Detected object openshift-console/console-56b489d6fc-b8dxl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:52.428786 3786 pods.go:176] Deleting pod: openshift-console/console-56b489d6fc-b8dxl I0428 15:38:52.430005 3786 pods.go:236] Attempting to release IPs for pod: openshift-console/console-56b489d6fc-b8dxl, ips: 10.128.0.25 I0428 15:38:52.430034 3786 obj_retry.go:498] Detected object openshift-console/console-56b489d6fc-b8dxl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:52.441118 3786 base_network_controller_pods.go:487] [default/openshift-console/console-5d45f7999c-wmx4k] creating logical port openshift-console_console-5d45f7999c-wmx4k for pod on switch ip-10-0-2-7.ec2.internal I0428 15:38:52.441294 3786 kube.go:256] Updating pod openshift-console/console-5d45f7999c-wmx4k I0428 15:38:52.460764 3786 pod.go:62] [openshift-console/console-5d45f7999c-wmx4k] pod update took 19.513406ms I0428 15:38:52.460797 3786 base_network_controller_pods.go:951] [default/openshift-console/console-5d45f7999c-wmx4k] addLogicalPort annotation time took 19.552215ms I0428 15:38:52.462084 3786 pods.go:271] [openshift-console/console-5d45f7999c-wmx4k] addLogicalPort took 21.007411ms, libovsdb time 889.299µs I0428 15:38:52.486208 3786 obj_retry.go:492] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:52.486232 3786 obj_retry.go:492] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:52.489581 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-796c59fd79-njh7k I0428 15:38:52.489608 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-796c59fd79-njh7k I0428 15:38:52.863005 3786 cni.go:452] [openshift-console/console-5d45f7999c-wmx4k c3228153527715775c04d4845c22033048baaa15546ad49688ed54f4796179db network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-5d45f7999c-wmx4k c3228153527715775c04d4845c22033048baaa15546ad49688ed54f4796179db network default NAD default NAD key ] I0428 15:38:52.873515 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-5d45f7999c-wmx4k, hostIfaceName: c32281535277157, network: default, NAD default, SandboxID: "c3228153527715775c04d4845c22033048baaa15546ad49688ed54f4796179db", PCI device ID: , UID: "dc26ee3a-05fd-47a2-b3a4-bae592923f48", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0428 15:38:52.932369 3786 cni.go:473] [openshift-console/console-5d45f7999c-wmx4k c3228153527715775c04d4845c22033048baaa15546ad49688ed54f4796179db network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-5d45f7999c-wmx4k c3228153527715775c04d4845c22033048baaa15546ad49688ed54f4796179db network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:a6:5d:e7:19:4f\",\"name\":\"c32281535277157\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/24edc160-47b5-473e-ade8-1f6655c541bc\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:38:53.235154 3786 obj_retry.go:492] Detected object openshift-console/console-56b489d6fc-b8dxl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:53.235182 3786 obj_retry.go:492] Detected object openshift-console/console-56b489d6fc-b8dxl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:53.239543 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-56b489d6fc-b8dxl I0428 15:38:53.239600 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-56b489d6fc-b8dxl W0428 15:39:03.277103 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0428 15:39:16.320743 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0428 15:39:16.320750 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0428 15:39:16.320805 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0428 15:39:17.321558 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0428 15:39:17.835298 3786 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-6cq6s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:17.836075 3786 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-6cq6s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:18.576323 3786 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-6cq6s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:18.576349 3786 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-6cq6s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:18.581442 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-957cdcb8f-6cq6s I0428 15:39:18.581467 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-957cdcb8f-6cq6s I0428 15:39:19.208525 3786 cni.go:452] [openshift-ingress/router-default-5f578f8577-xnnpr c3986e0fd8518323fce62924225b4b9bd82709a1702cdbc38ca257a2e6f43d66 network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-5f578f8577-xnnpr c3986e0fd8518323fce62924225b4b9bd82709a1702cdbc38ca257a2e6f43d66 network default NAD default NAD key ] I0428 15:39:19.309681 3786 cni.go:473] [openshift-ingress/router-default-5f578f8577-xnnpr c3986e0fd8518323fce62924225b4b9bd82709a1702cdbc38ca257a2e6f43d66 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-5f578f8577-xnnpr c3986e0fd8518323fce62924225b4b9bd82709a1702cdbc38ca257a2e6f43d66 network default NAD default NAD key default], result "{}", err I0428 15:39:19.358839 3786 obj_retry.go:498] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:19.358877 3786 pods.go:176] Deleting pod: openshift-ingress/router-default-5f578f8577-xnnpr I0428 15:39:19.360121 3786 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-5f578f8577-xnnpr, ips: 10.128.0.19 I0428 15:39:19.360148 3786 obj_retry.go:498] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:20.328941 3786 obj_retry.go:492] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:20.328966 3786 obj_retry.go:492] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:20.338071 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5f578f8577-xnnpr I0428 15:39:20.338115 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5f578f8577-xnnpr I0428 15:39:33.166163 3786 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0428 15:39:41.097010 3786 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:41.097592 3786 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:41.663942 3786 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:41.663966 3786 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:41.671398 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0428 15:39:41.671425 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0428 15:39:44.913318 3786 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:44.913857 3786 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:45.688136 3786 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:45.688159 3786 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:45.693713 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0428 15:39:45.693739 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0428 15:40:00.232149 3786 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29623180-zgjmh for pod on switch ip-10-0-2-7.ec2.internal I0428 15:40:00.232264 3786 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh I0428 15:40:00.251688 3786 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh] pod update took 19.461713ms I0428 15:40:00.251718 3786 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh] addLogicalPort annotation time took 19.495056ms I0428 15:40:00.253002 3786 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh] addLogicalPort took 20.872722ms, libovsdb time 933.702µs I0428 15:40:00.607160 3786 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh 041fb39a541fb428aceff2cd825512714227c9991f6fd5e27551afa82502a4bb network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh 041fb39a541fb428aceff2cd825512714227c9991f6fd5e27551afa82502a4bb network default NAD default NAD key ] I0428 15:40:00.617001 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29623180-zgjmh, hostIfaceName: 041fb39a541fb42, network: default, NAD default, SandboxID: "041fb39a541fb428aceff2cd825512714227c9991f6fd5e27551afa82502a4bb", PCI device ID: , UID: "42438da7-6373-4de0-9a30-7e9c645aa479", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0428 15:40:00.671151 3786 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh 041fb39a541fb428aceff2cd825512714227c9991f6fd5e27551afa82502a4bb network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh 041fb39a541fb428aceff2cd825512714227c9991f6fd5e27551afa82502a4bb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:7b:0a:03:7d:79\",\"name\":\"041fb39a541fb42\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/780e6b76-fc4c-4916-b325-69b1a0eea80c\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:40:03.445829 3786 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:03.446422 3786 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:03.778508 3786 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:03.778539 3786 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:03.785625 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0428 15:40:03.785651 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0428 15:40:05.528903 3786 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh 041fb39a541fb428aceff2cd825512714227c9991f6fd5e27551afa82502a4bb network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh 041fb39a541fb428aceff2cd825512714227c9991f6fd5e27551afa82502a4bb network default NAD default NAD key ] I0428 15:40:05.618621 3786 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh 041fb39a541fb428aceff2cd825512714227c9991f6fd5e27551afa82502a4bb network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh 041fb39a541fb428aceff2cd825512714227c9991f6fd5e27551afa82502a4bb network default NAD default NAD key default], result "{}", err I0428 15:40:05.662050 3786 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:05.662102 3786 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh I0428 15:40:05.663315 3786 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh, ips: 10.128.0.28 I0428 15:40:05.663339 3786 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:06.547933 3786 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:06.547958 3786 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:10.501335 3786 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-zgjjj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:10.501897 3786 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-zgjjj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:10.794018 3786 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-zgjjj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:10.794043 3786 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-zgjjj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:10.799502 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5d45f7999c-zgjjj I0428 15:40:10.799526 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5d45f7999c-zgjjj I0428 15:40:20.945545 3786 cni.go:452] [openshift-console/console-5d45f7999c-wmx4k c3228153527715775c04d4845c22033048baaa15546ad49688ed54f4796179db network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-5d45f7999c-wmx4k c3228153527715775c04d4845c22033048baaa15546ad49688ed54f4796179db network default NAD default NAD key ] I0428 15:40:21.033788 3786 cni.go:473] [openshift-console/console-5d45f7999c-wmx4k c3228153527715775c04d4845c22033048baaa15546ad49688ed54f4796179db network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-5d45f7999c-wmx4k c3228153527715775c04d4845c22033048baaa15546ad49688ed54f4796179db network default NAD default NAD key default], result "{}", err I0428 15:40:21.085963 3786 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-wmx4k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:21.086002 3786 pods.go:176] Deleting pod: openshift-console/console-5d45f7999c-wmx4k I0428 15:40:21.087227 3786 pods.go:236] Attempting to release IPs for pod: openshift-console/console-5d45f7999c-wmx4k, ips: 10.128.0.27 I0428 15:40:21.087258 3786 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-wmx4k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:21.586959 3786 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-wmx4k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:21.586990 3786 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-wmx4k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:21.593106 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5d45f7999c-wmx4k I0428 15:40:21.593131 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5d45f7999c-wmx4k I0428 15:40:48.294634 3786 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:48.295183 3786 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:48.951703 3786 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:48.951733 3786 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:48.959753 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0428 15:40:48.959778 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0428 15:42:37.197211 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0428 15:42:49.202613 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0428 15:43:00.096465 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0428 15:43:17.192222 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0428 15:43:20.187434 3786 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0428 15:43:36.468699 3786 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-xj5ts] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-5d7684fb5d-xj5ts for pod on switch ip-10-0-2-7.ec2.internal I0428 15:43:36.468841 3786 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-xj5ts I0428 15:43:36.497944 3786 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-xj5ts] pod update took 29.132667ms I0428 15:43:36.497971 3786 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-xj5ts] addLogicalPort annotation time took 29.165322ms I0428 15:43:36.499221 3786 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-xj5ts] addLogicalPort took 30.541424ms, libovsdb time 885.474µs I0428 15:43:36.826937 3786 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-xj5ts f36f6b3b17e2a403d07adec739eae8840e70060c262edef1ab81fb3035fd2d14 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-xj5ts f36f6b3b17e2a403d07adec739eae8840e70060c262edef1ab81fb3035fd2d14 network default NAD default NAD key ] I0428 15:43:36.836164 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-5d7684fb5d-xj5ts, hostIfaceName: f36f6b3b17e2a40, network: default, NAD default, SandboxID: "f36f6b3b17e2a403d07adec739eae8840e70060c262edef1ab81fb3035fd2d14", PCI device ID: , UID: "9d0ea038-34d6-4edf-82bb-c6819e4907da", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0428 15:43:36.888118 3786 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-xj5ts f36f6b3b17e2a403d07adec739eae8840e70060c262edef1ab81fb3035fd2d14 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-5d7684fb5d-xj5ts f36f6b3b17e2a403d07adec739eae8840e70060c262edef1ab81fb3035fd2d14 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:88:00:3a:dc:dd\",\"name\":\"f36f6b3b17e2a40\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f3a23d35-1ebb-4084-9587-d0fe16ef7cac\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:43:39.338006 3786 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:43:48.219200 3786 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 W0428 15:43:50.351050 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:43:50.391624 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0428 15:43:50.391693 3786 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-8j7mw IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-8j7mw: no pod IPs found I0428 15:44:22.207933 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0428 15:44:39.225279 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0428 15:44:43.099531 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=71 I0428 15:45:01.188247 3786 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0428 15:45:41.100645 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=139 I0428 15:45:47.761700 3786 namespace.go:98] [tsf] adding namespace I0428 15:45:47.762613 3786 namespace.go:102] [tsf] adding namespace took 889.246µs I0428 15:45:47.775796 3786 namespace.go:142] [tsf] updating namespace I0428 15:45:47.791499 3786 namespace.go:142] [tsf] updating namespace I0428 15:45:51.427842 3786 namespace.go:98] [cert-manager-operator] adding namespace I0428 15:45:51.428832 3786 namespace.go:102] [cert-manager-operator] adding namespace took 954.206µs I0428 15:45:51.436079 3786 namespace.go:98] [konflux-ui] adding namespace I0428 15:45:51.436954 3786 namespace.go:102] [konflux-ui] adding namespace took 834.909µs I0428 15:45:51.437024 3786 namespace.go:98] [tsf-tpa] adding namespace I0428 15:45:51.437024 3786 namespace.go:98] [rhtpa-operator] adding namespace I0428 15:45:51.437748 3786 namespace.go:102] [tsf-tpa] adding namespace took 706.403µs I0428 15:45:51.438498 3786 namespace.go:102] [rhtpa-operator] adding namespace took 1.384165ms I0428 15:45:51.452017 3786 namespace.go:98] [rhbk-operator] adding namespace I0428 15:45:51.452129 3786 namespace.go:98] [tsf-keycloak] adding namespace I0428 15:45:51.452303 3786 namespace.go:98] [konflux-operator] adding namespace I0428 15:45:51.452894 3786 namespace.go:102] [rhbk-operator] adding namespace took 846.996µs I0428 15:45:51.453538 3786 namespace.go:102] [tsf-keycloak] adding namespace took 1.392514ms I0428 15:45:51.453564 3786 namespace.go:98] [tsf-tas] adding namespace I0428 15:45:51.454131 3786 namespace.go:102] [tsf-tas] adding namespace took 560.117µs I0428 15:45:51.454695 3786 namespace.go:102] [konflux-operator] adding namespace took 2.373085ms I0428 15:45:51.470765 3786 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:45:51.494527 3786 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:45:51.505791 3786 namespace.go:142] [konflux-ui] updating namespace I0428 15:45:51.506341 3786 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:45:51.506438 3786 namespace.go:142] [rhbk-operator] updating namespace I0428 15:45:51.511151 3786 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:51.515035 3786 namespace.go:142] [konflux-ui] updating namespace I0428 15:45:51.519922 3786 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:51.533682 3786 namespace.go:142] [konflux-ui] updating namespace I0428 15:45:51.533823 3786 namespace.go:142] [tsf-tas] updating namespace I0428 15:45:51.533948 3786 namespace.go:142] [konflux-operator] updating namespace I0428 15:45:51.533956 3786 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:45:51.541843 3786 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:51.552142 3786 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:51.561230 3786 namespace.go:142] [rhbk-operator] updating namespace I0428 15:45:51.571860 3786 namespace.go:142] [rhbk-operator] updating namespace I0428 15:45:51.584702 3786 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:51.595288 3786 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:51.599575 3786 namespace.go:142] [tsf-tas] updating namespace I0428 15:45:51.608657 3786 namespace.go:142] [tsf-tas] updating namespace I0428 15:45:51.616634 3786 namespace.go:142] [konflux-operator] updating namespace I0428 15:45:51.628410 3786 namespace.go:142] [konflux-operator] updating namespace I0428 15:45:51.634315 3786 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:45:51.642546 3786 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:45:52.793217 3786 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:52.807204 3786 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:53.225519 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0428 15:45:58.213125 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0428 15:45:58.291660 3786 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.292315 3786 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.292348 3786 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.292356 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:45:58.292364 3786 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.292368 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:45:59.322721 3786 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:59.322749 3786 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:59.322766 3786 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:59.322777 3786 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:00.327144 3786 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.327712 3786 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.327738 3786 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.327745 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:00.327753 3786 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.327757 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:01.360387 3786 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:01.360406 3786 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:01.360415 3786 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:01.360424 3786 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:03.893487 3786 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.894038 3786 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.894069 3786 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.894083 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:03.894116 3786 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.894122 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:04.930524 3786 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:04.930551 3786 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:04.930572 3786 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:04.930581 3786 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:07.643969 3786 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.644565 3786 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.644594 3786 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.644604 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:07.644615 3786 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.644622 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:07.746823 3786 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.747387 3786 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.747412 3786 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.747419 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:07.747428 3786 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.747432 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:08.697422 3786 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.697454 3786 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.697472 3786 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.697481 3786 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.846523 3786 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.846554 3786 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.846575 3786 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.846583 3786 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:09.915212 3786 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-tz2ht] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-tz2ht for pod on switch ip-10-0-2-7.ec2.internal I0428 15:46:09.915358 3786 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-tz2ht I0428 15:46:09.938678 3786 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-tz2ht] pod update took 23.346639ms I0428 15:46:09.938708 3786 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-tz2ht] addLogicalPort annotation time took 23.382246ms I0428 15:46:09.939909 3786 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-tz2ht] addLogicalPort took 24.722862ms, libovsdb time 875.513µs I0428 15:46:10.277358 3786 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-tz2ht 06032afc237d3849652f9aa2fc409a64698319a79354f4a1912e872694bcf877 network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-tz2ht 06032afc237d3849652f9aa2fc409a64698319a79354f4a1912e872694bcf877 network default NAD default NAD key ] I0428 15:46:10.286942 3786 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-tz2ht, hostIfaceName: 06032afc237d384, network: default, NAD default, SandboxID: "06032afc237d3849652f9aa2fc409a64698319a79354f4a1912e872694bcf877", PCI device ID: , UID: "7fc5f548-5a8f-47c8-ac4e-6c8c82f936ae", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0428 15:46:10.340786 3786 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-tz2ht 06032afc237d3849652f9aa2fc409a64698319a79354f4a1912e872694bcf877 network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-tz2ht 06032afc237d3849652f9aa2fc409a64698319a79354f4a1912e872694bcf877 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:52:cc:2f:58:83\",\"name\":\"06032afc237d384\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/658d6f51-5fdf-4753-8ed1-45b1a4a0ca04\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:46:10.478428 3786 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.479001 3786 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.479028 3786 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.479037 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:10.479046 3786 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.479050 3786 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:11.494121 3786 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:11.494146 3786 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:11.494165 3786 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:11.494174 3786 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:13.711341 3786 namespace.go:98] [cert-manager] adding namespace I0428 15:46:13.712212 3786 namespace.go:102] [cert-manager] adding namespace took 845.321µs I0428 15:46:13.744430 3786 namespace.go:142] [cert-manager] updating namespace I0428 15:46:13.752356 3786 namespace.go:142] [cert-manager] updating namespace I0428 15:46:17.609576 3786 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-cainjector-5d99449ffd-9rbc7] creating logical port cert-manager_cert-manager-cainjector-5d99449ffd-9rbc7 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:46:17.609720 3786 kube.go:256] Updating pod cert-manager/cert-manager-cainjector-5d99449ffd-9rbc7 I0428 15:46:17.627225 3786 pod.go:62] [cert-manager/cert-manager-cainjector-5d99449ffd-9rbc7] pod update took 17.532906ms I0428 15:46:17.627252 3786 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-cainjector-5d99449ffd-9rbc7] addLogicalPort annotation time took 17.565957ms I0428 15:46:17.628612 3786 pods.go:271] [cert-manager/cert-manager-cainjector-5d99449ffd-9rbc7] addLogicalPort took 19.054826ms, libovsdb time 896.185µs I0428 15:46:17.984533 3786 cni.go:452] [cert-manager/cert-manager-cainjector-5d99449ffd-9rbc7 95f7fd102f64dc640331747c390bcfe929568acadd4958969abd2de793b962ed network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-9rbc7 95f7fd102f64dc640331747c390bcfe929568acadd4958969abd2de793b962ed network default NAD default NAD key ] I0428 15:46:17.993982 3786 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-cainjector-5d99449ffd-9rbc7, hostIfaceName: 95f7fd102f64dc6, network: default, NAD default, SandboxID: "95f7fd102f64dc640331747c390bcfe929568acadd4958969abd2de793b962ed", PCI device ID: , UID: "4ef78a47-b5f0-44fb-9e10-476e3043d5fc", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0428 15:46:18.046788 3786 cni.go:473] [cert-manager/cert-manager-cainjector-5d99449ffd-9rbc7 95f7fd102f64dc640331747c390bcfe929568acadd4958969abd2de793b962ed network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-cainjector-5d99449ffd-9rbc7 95f7fd102f64dc640331747c390bcfe929568acadd4958969abd2de793b962ed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:44:4c:87:ce:2e\",\"name\":\"95f7fd102f64dc6\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe378e61-f259-4f5d-8ef6-9a59cbb48a32\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:46:18.620598 3786 base_network_controller_pods.go:487] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-bzlf5] creating logical port tsf-tpa_rhtpa-operator-controller-manager-586cbb496b-bzlf5 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:46:18.620765 3786 kube.go:256] Updating pod tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-bzlf5 I0428 15:46:18.641500 3786 pod.go:62] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-bzlf5] pod update took 20.76217ms I0428 15:46:18.641524 3786 base_network_controller_pods.go:951] [default/tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-bzlf5] addLogicalPort annotation time took 20.791051ms I0428 15:46:18.642975 3786 pods.go:271] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-bzlf5] addLogicalPort took 22.393542ms, libovsdb time 1.021217ms I0428 15:46:18.983481 3786 cni.go:452] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-bzlf5 803ddd939d0d741f2a8750d6d8e97b478b97aecdafb9ee36c33f888329b3bf9c network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-bzlf5 803ddd939d0d741f2a8750d6d8e97b478b97aecdafb9ee36c33f888329b3bf9c network default NAD default NAD key ] I0428 15:46:18.993336 3786 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: rhtpa-operator-controller-manager-586cbb496b-bzlf5, hostIfaceName: 803ddd939d0d741, network: default, NAD default, SandboxID: "803ddd939d0d741f2a8750d6d8e97b478b97aecdafb9ee36c33f888329b3bf9c", PCI device ID: , UID: "f2c6f576-62c2-42b8-848f-d35bf8d02861", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0428 15:46:19.048495 3786 cni.go:473] [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-bzlf5 803ddd939d0d741f2a8750d6d8e97b478b97aecdafb9ee36c33f888329b3bf9c network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/rhtpa-operator-controller-manager-586cbb496b-bzlf5 803ddd939d0d741f2a8750d6d8e97b478b97aecdafb9ee36c33f888329b3bf9c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:90:94:06:92:a2\",\"name\":\"803ddd939d0d741\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/27124207-aa17-4d54-b7ae-09e316117303\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:46:19.238630 3786 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 15:46:19.238656 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 15:46:19.274752 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.884461 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.891681 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.929903 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.938157 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0428 15:46:24.107695 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=756 I0428 15:46:28.679397 3786 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59c77684cc-xvwtb] creating logical port cert-manager_cert-manager-59c77684cc-xvwtb for pod on switch ip-10-0-2-7.ec2.internal I0428 15:46:28.679493 3786 kube.go:256] Updating pod cert-manager/cert-manager-59c77684cc-xvwtb I0428 15:46:28.697023 3786 pod.go:62] [cert-manager/cert-manager-59c77684cc-xvwtb] pod update took 17.549413ms I0428 15:46:28.697056 3786 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-59c77684cc-xvwtb] addLogicalPort annotation time took 17.58788ms I0428 15:46:28.698292 3786 pods.go:271] [cert-manager/cert-manager-59c77684cc-xvwtb] addLogicalPort took 18.909304ms, libovsdb time 870.283µs I0428 15:46:29.040173 3786 cni.go:452] [cert-manager/cert-manager-59c77684cc-xvwtb 87e4c3ca1411b4b8a963140dc0060fc1c5bb44855713e67f1e1518469de7c29f network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-59c77684cc-xvwtb 87e4c3ca1411b4b8a963140dc0060fc1c5bb44855713e67f1e1518469de7c29f network default NAD default NAD key ] I0428 15:46:29.049991 3786 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59c77684cc-xvwtb, hostIfaceName: 87e4c3ca1411b4b, network: default, NAD default, SandboxID: "87e4c3ca1411b4b8a963140dc0060fc1c5bb44855713e67f1e1518469de7c29f", PCI device ID: , UID: "e12e8302-bb9f-4786-824a-d560b3a9c6c6", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0428 15:46:29.105836 3786 cni.go:473] [cert-manager/cert-manager-59c77684cc-xvwtb 87e4c3ca1411b4b8a963140dc0060fc1c5bb44855713e67f1e1518469de7c29f network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-59c77684cc-xvwtb 87e4c3ca1411b4b8a963140dc0060fc1c5bb44855713e67f1e1518469de7c29f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:94:98:e2:78:27\",\"name\":\"87e4c3ca1411b4b\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/883a110c-3ad9-4e18-b59a-e0d16c01ed48\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:46:30.107460 3786 namespace.go:142] [openshift-operators] updating namespace I0428 15:46:30.115786 3786 namespace.go:142] [openshift-operators] updating namespace W0428 15:46:36.094369 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0428 15:46:37.479247 3786 namespace.go:142] [openshift-operators] updating namespace I0428 15:46:37.732652 3786 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-xddtx] creating logical port openshift-operators_openshift-pipelines-operator-6bdf6c8794-xddtx for pod on switch ip-10-0-2-7.ec2.internal I0428 15:46:37.732761 3786 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-xddtx I0428 15:46:37.778502 3786 pod.go:62] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-xddtx] pod update took 45.765849ms I0428 15:46:37.778526 3786 base_network_controller_pods.go:951] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-xddtx] addLogicalPort annotation time took 45.794652ms I0428 15:46:37.779569 3786 pods.go:271] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-xddtx] addLogicalPort took 46.932961ms, libovsdb time 722.527µs I0428 15:46:38.090549 3786 cni.go:452] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-xddtx d86d1f4b9aab2fc8811a194d0574d6e12dbbb88dd2a2771e95361581722bfff7 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-xddtx d86d1f4b9aab2fc8811a194d0574d6e12dbbb88dd2a2771e95361581722bfff7 network default NAD default NAD key ] I0428 15:46:38.100400 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6bdf6c8794-xddtx, hostIfaceName: d86d1f4b9aab2fc, network: default, NAD default, SandboxID: "d86d1f4b9aab2fc8811a194d0574d6e12dbbb88dd2a2771e95361581722bfff7", PCI device ID: , UID: "e5594b8a-cbf5-4735-bb8a-e1fcb99c8df5", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0428 15:46:38.102113 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=78 I0428 15:46:38.153765 3786 cni.go:473] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-xddtx d86d1f4b9aab2fc8811a194d0574d6e12dbbb88dd2a2771e95361581722bfff7 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-xddtx d86d1f4b9aab2fc8811a194d0574d6e12dbbb88dd2a2771e95361581722bfff7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:a8:c6:02:01:07\",\"name\":\"d86d1f4b9aab2fc\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/62f161b6-73bf-49af-ab36-14ead6f63b93\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:46:44.976026 3786 namespace.go:98] [trusted-artifact-signer] adding namespace I0428 15:46:44.976901 3786 namespace.go:102] [trusted-artifact-signer] adding namespace took 854.686µs I0428 15:46:45.004224 3786 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:45.025203 3786 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:45.112953 3786 base_network_controller_pods.go:487] [default/trusted-artifact-signer/cli-server-7d56d9fd65-kqsfq] creating logical port trusted-artifact-signer_cli-server-7d56d9fd65-kqsfq for pod on switch ip-10-0-2-7.ec2.internal I0428 15:46:45.113083 3786 kube.go:256] Updating pod trusted-artifact-signer/cli-server-7d56d9fd65-kqsfq I0428 15:46:45.151503 3786 pod.go:62] [trusted-artifact-signer/cli-server-7d56d9fd65-kqsfq] pod update took 38.444712ms I0428 15:46:45.151529 3786 base_network_controller_pods.go:951] [default/trusted-artifact-signer/cli-server-7d56d9fd65-kqsfq] addLogicalPort annotation time took 38.476865ms I0428 15:46:45.152744 3786 pods.go:271] [trusted-artifact-signer/cli-server-7d56d9fd65-kqsfq] addLogicalPort took 39.811967ms, libovsdb time 851.692µs W0428 15:46:45.406755 3786 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0428 15:46:45.473069 3786 cni.go:452] [trusted-artifact-signer/cli-server-7d56d9fd65-kqsfq 21a57e4363a48ce47aa595ac4de22a3031d4bd1eb91f216d573745f9bf03f1cf network default NAD default NAD key ] ADD starting CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-kqsfq 21a57e4363a48ce47aa595ac4de22a3031d4bd1eb91f216d573745f9bf03f1cf network default NAD default NAD key ] I0428 15:46:45.483597 3786 helper_linux.go:556] ConfigureOVS: namespace: trusted-artifact-signer, podName: cli-server-7d56d9fd65-kqsfq, hostIfaceName: 21a57e4363a48ce, network: default, NAD default, SandboxID: "21a57e4363a48ce47aa595ac4de22a3031d4bd1eb91f216d573745f9bf03f1cf", PCI device ID: , UID: "8f0a5f88-3b1d-44bc-b0c0-1133efa3b9af", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0428 15:46:45.539410 3786 cni.go:473] [trusted-artifact-signer/cli-server-7d56d9fd65-kqsfq 21a57e4363a48ce47aa595ac4de22a3031d4bd1eb91f216d573745f9bf03f1cf network default NAD default NAD key default] ADD finished CNI request [trusted-artifact-signer/cli-server-7d56d9fd65-kqsfq 21a57e4363a48ce47aa595ac4de22a3031d4bd1eb91f216d573745f9bf03f1cf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:99:c9:f1:0d:59\",\"name\":\"21a57e4363a48ce\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/959ef77d-1654-456c-b47c-d96ac309a652\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:46:47.062722 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0428 15:46:50.661578 3786 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:50.662161 3786 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:50.681214 3786 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:50.681242 3786 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:50.691350 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0428 15:46:50.691364 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0428 15:46:53.213535 3786 namespace.go:98] [openshift-pipelines] adding namespace I0428 15:46:53.214563 3786 namespace.go:102] [openshift-pipelines] adding namespace took 998.467µs I0428 15:46:53.234681 3786 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:46:53.940598 3786 namespace.go:142] [cert-manager] updating namespace I0428 15:46:54.142900 3786 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:46:54.547720 3786 namespace.go:142] [konflux-operator] updating namespace I0428 15:46:54.746766 3786 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:54.947751 3786 namespace.go:142] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] updating namespace I0428 15:46:55.101965 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=71 I0428 15:46:55.350539 3786 namespace.go:142] [rhbk-operator] updating namespace I0428 15:46:55.552600 3786 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:46:55.751911 3786 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:55.952251 3786 namespace.go:142] [tsf] updating namespace I0428 15:46:56.155057 3786 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:46:56.355338 3786 namespace.go:142] [tsf-tas] updating namespace I0428 15:46:56.557884 3786 namespace.go:142] [tsf-tpa] updating namespace I0428 15:46:56.690723 3786 namespace.go:142] [cert-manager] updating namespace I0428 15:46:56.702544 3786 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:46:56.716290 3786 namespace.go:142] [default] updating namespace I0428 15:46:56.728899 3786 namespace.go:142] [konflux-operator] updating namespace I0428 15:46:56.739284 3786 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:56.754017 3786 namespace.go:142] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] updating namespace I0428 15:46:56.769395 3786 namespace.go:142] [openshift] updating namespace I0428 15:46:56.786227 3786 namespace.go:142] [rhbk-operator] updating namespace I0428 15:46:56.807505 3786 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:46:56.830009 3786 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:56.847813 3786 namespace.go:142] [tsf] updating namespace I0428 15:46:56.865305 3786 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:46:56.878913 3786 namespace.go:142] [tsf-tas] updating namespace I0428 15:46:56.915234 3786 namespace.go:142] [tsf-tpa] updating namespace I0428 15:46:57.014186 3786 namespace.go:142] [cert-manager] updating namespace I0428 15:46:57.103955 3786 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:46:57.181301 3786 namespace.go:142] [default] updating namespace I0428 15:46:57.256629 3786 namespace.go:142] [konflux-operator] updating namespace I0428 15:46:57.352366 3786 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:57.456558 3786 namespace.go:142] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] updating namespace I0428 15:46:57.556295 3786 namespace.go:142] [openshift] updating namespace I0428 15:46:57.654553 3786 namespace.go:142] [rhbk-operator] updating namespace I0428 15:46:57.748457 3786 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:46:57.846296 3786 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:57.945998 3786 namespace.go:142] [tsf] updating namespace I0428 15:46:58.054326 3786 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:46:58.154476 3786 namespace.go:142] [tsf-tas] updating namespace I0428 15:46:58.249336 3786 namespace.go:142] [tsf-tpa] updating namespace I0428 15:46:58.663143 3786 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:47:06.276670 3786 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-5w8dk] creating logical port openshift-pipelines_tekton-events-controller-57d7f78b8d-5w8dk for pod on switch ip-10-0-2-7.ec2.internal I0428 15:47:06.276773 3786 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-57d7f78b8d-5w8dk I0428 15:47:06.298285 3786 pod.go:62] [openshift-pipelines/tekton-events-controller-57d7f78b8d-5w8dk] pod update took 21.536359ms I0428 15:47:06.298308 3786 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-5w8dk] addLogicalPort annotation time took 21.565412ms I0428 15:47:06.299488 3786 pods.go:271] [openshift-pipelines/tekton-events-controller-57d7f78b8d-5w8dk] addLogicalPort took 22.838075ms, libovsdb time 762.998µs W0428 15:47:06.603440 3786 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0428 15:47:06.630944 3786 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0428 15:47:06.637012 3786 cni.go:452] [openshift-pipelines/tekton-events-controller-57d7f78b8d-5w8dk 3c130bb6daf96fedc2231caf461562e98767879022b10487dc1b96b49625ec74 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-5w8dk 3c130bb6daf96fedc2231caf461562e98767879022b10487dc1b96b49625ec74 network default NAD default NAD key ] I0428 15:47:06.647334 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-57d7f78b8d-5w8dk, hostIfaceName: 3c130bb6daf96fe, network: default, NAD default, SandboxID: "3c130bb6daf96fedc2231caf461562e98767879022b10487dc1b96b49625ec74", PCI device ID: , UID: "5ac8d71b-b8d0-47f7-a94b-b91b143ee28b", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0428 15:47:06.702916 3786 cni.go:473] [openshift-pipelines/tekton-events-controller-57d7f78b8d-5w8dk 3c130bb6daf96fedc2231caf461562e98767879022b10487dc1b96b49625ec74 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-5w8dk 3c130bb6daf96fedc2231caf461562e98767879022b10487dc1b96b49625ec74 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:5a:3e:6d:bf:b3\",\"name\":\"3c130bb6daf96fe\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/db8650a4-6bb1-4a9e-9682-e7864447d055\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:47:10.210710 3786 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:10.210739 3786 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:10.210750 3786 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:10.488007 3786 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:10.488026 3786 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:10.488031 3786 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:10.488035 3786 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:13.506222 3786 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:13.506241 3786 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:13.506246 3786 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:21.216869 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:21.216892 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:21.216898 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0428 15:47:31.422928 3786 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-z7w85] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-z7w85 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:47:31.423028 3786 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-z7w85 I0428 15:47:31.435238 3786 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-z7w85] pod update took 12.23754ms I0428 15:47:31.435259 3786 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-z7w85] addLogicalPort annotation time took 12.26383ms I0428 15:47:31.436482 3786 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-z7w85] addLogicalPort took 13.574355ms, libovsdb time 817.306µs I0428 15:47:31.973988 3786 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-z7w85 9fb9661fabf51182bd42656b4274588b1a36d086baed4de3da57e95e4516c59d network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-z7w85 9fb9661fabf51182bd42656b4274588b1a36d086baed4de3da57e95e4516c59d network default NAD default NAD key ] I0428 15:47:31.983454 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-z7w85, hostIfaceName: 9fb9661fabf5118, network: default, NAD default, SandboxID: "9fb9661fabf51182bd42656b4274588b1a36d086baed4de3da57e95e4516c59d", PCI device ID: , UID: "589cdc29-da29-4731-8e2b-16c9c4acd744", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0428 15:47:32.039235 3786 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-z7w85 9fb9661fabf51182bd42656b4274588b1a36d086baed4de3da57e95e4516c59d network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-z7w85 9fb9661fabf51182bd42656b4274588b1a36d086baed4de3da57e95e4516c59d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:36:4a:d3:c8:2f\",\"name\":\"9fb9661fabf5118\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8f38ffe9-f092-4931-a2c0-f73a5afa62cf\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:47:36.326990 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0428 15:47:36.891569 3786 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:36.892119 3786 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:36.912384 3786 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:36.912401 3786 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:36.915406 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0428 15:47:36.915430 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0428 15:47:38.654639 3786 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0428 15:47:38.655357 3786 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0428 15:47:38.655378 3786 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0428 15:47:38.657303 3786 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0428 15:47:46.815672 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0428 15:47:46.815691 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0428 15:47:46.816880 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 15:47:46.816897 3786 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 15:47:47.346939 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0428 15:47:49.280968 3786 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-chains-controller-55fb698565-8dvcp] creating logical port openshift-pipelines_tekton-chains-controller-55fb698565-8dvcp for pod on switch ip-10-0-2-7.ec2.internal I0428 15:47:49.281122 3786 kube.go:256] Updating pod openshift-pipelines/tekton-chains-controller-55fb698565-8dvcp I0428 15:47:49.295661 3786 pod.go:62] [openshift-pipelines/tekton-chains-controller-55fb698565-8dvcp] pod update took 14.581416ms I0428 15:47:49.295690 3786 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-chains-controller-55fb698565-8dvcp] addLogicalPort annotation time took 14.613655ms I0428 15:47:49.296966 3786 pods.go:271] [openshift-pipelines/tekton-chains-controller-55fb698565-8dvcp] addLogicalPort took 16.016618ms, libovsdb time 884.654µs I0428 15:47:49.640822 3786 cni.go:452] [openshift-pipelines/tekton-chains-controller-55fb698565-8dvcp 37a976034259f60d25894eca5c3cd1fe49d5b6e10cc0885f43449e36ebf2897a network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-8dvcp 37a976034259f60d25894eca5c3cd1fe49d5b6e10cc0885f43449e36ebf2897a network default NAD default NAD key ] I0428 15:47:49.651287 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-chains-controller-55fb698565-8dvcp, hostIfaceName: 37a976034259f60, network: default, NAD default, SandboxID: "37a976034259f60d25894eca5c3cd1fe49d5b6e10cc0885f43449e36ebf2897a", PCI device ID: , UID: "77712997-f4b5-4a8d-a387-b450b1a35c8b", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0428 15:47:49.706069 3786 cni.go:473] [openshift-pipelines/tekton-chains-controller-55fb698565-8dvcp 37a976034259f60d25894eca5c3cd1fe49d5b6e10cc0885f43449e36ebf2897a network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-chains-controller-55fb698565-8dvcp 37a976034259f60d25894eca5c3cd1fe49d5b6e10cc0885f43449e36ebf2897a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:72:fc:73:d2:23\",\"name\":\"37a976034259f60\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dbe1a620-edb0-4a09-802d-2e9f675200a5\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:47:54.268827 3786 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 15:47:57.348471 3786 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:47:57.348600 3786 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0428 15:47:57.360602 3786 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 12.027279ms I0428 15:47:57.360624 3786 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 12.055899ms I0428 15:47:57.362046 3786 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 13.590379ms, libovsdb time 952.989µs I0428 15:48:00.408596 3786 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 85390832160cf7a2fb3e1541908a8c3db0f031839a24f542ad7418d95823480c network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 85390832160cf7a2fb3e1541908a8c3db0f031839a24f542ad7418d95823480c network default NAD default NAD key ] I0428 15:48:00.418529 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 85390832160cf7a, network: default, NAD default, SandboxID: "85390832160cf7a2fb3e1541908a8c3db0f031839a24f542ad7418d95823480c", PCI device ID: , UID: "d6106685-1cce-49a8-b1a5-2d20b44a8950", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0428 15:48:00.472281 3786 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 85390832160cf7a2fb3e1541908a8c3db0f031839a24f542ad7418d95823480c network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 85390832160cf7a2fb3e1541908a8c3db0f031839a24f542ad7418d95823480c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:b6:27:2c:c0:4c\",\"name\":\"85390832160cf7a\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4f97c0d9-9355-48e0-a6c6-ef352f763cbb\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:48:06.507062 3786 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 15:48:08.054319 3786 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-qk6fr] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-qk6fr for pod on switch ip-10-0-2-7.ec2.internal I0428 15:48:08.054420 3786 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-qk6fr I0428 15:48:08.068948 3786 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-qk6fr] pod update took 14.551324ms I0428 15:48:08.068972 3786 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-qk6fr] addLogicalPort annotation time took 14.581698ms I0428 15:48:08.070214 3786 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-qk6fr] addLogicalPort took 15.906208ms, libovsdb time 801.617µs I0428 15:48:08.414943 3786 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-qk6fr 3e6aa57d03a1f8dc464f114e100c5bb7c0b1342da320ee7c608ad7b3b3eb2ac5 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-qk6fr 3e6aa57d03a1f8dc464f114e100c5bb7c0b1342da320ee7c608ad7b3b3eb2ac5 network default NAD default NAD key ] I0428 15:48:08.424311 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-qk6fr, hostIfaceName: 3e6aa57d03a1f8d, network: default, NAD default, SandboxID: "3e6aa57d03a1f8dc464f114e100c5bb7c0b1342da320ee7c608ad7b3b3eb2ac5", PCI device ID: , UID: "cd317ec0-9df6-4d89-848b-09086752d6c1", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0428 15:48:08.478393 3786 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-qk6fr 3e6aa57d03a1f8dc464f114e100c5bb7c0b1342da320ee7c608ad7b3b3eb2ac5 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-qk6fr 3e6aa57d03a1f8dc464f114e100c5bb7c0b1342da320ee7c608ad7b3b3eb2ac5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:cc:0d:d7:18:eb\",\"name\":\"3e6aa57d03a1f8d\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/809f339b-4d0f-4027-a0c4-ea5541536388\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:48:13.797457 3786 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:48:13.797477 3786 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:48:13.797484 3786 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0428 15:48:20.460169 3786 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:48:24.768801 3786 base_network_controller_pods.go:487] [default/openshift-pipelines/tkn-cli-serve-784b566968-5h6jc] creating logical port openshift-pipelines_tkn-cli-serve-784b566968-5h6jc for pod on switch ip-10-0-2-7.ec2.internal I0428 15:48:24.768914 3786 kube.go:256] Updating pod openshift-pipelines/tkn-cli-serve-784b566968-5h6jc I0428 15:48:24.777405 3786 pod.go:62] [openshift-pipelines/tkn-cli-serve-784b566968-5h6jc] pod update took 8.520952ms I0428 15:48:24.777429 3786 base_network_controller_pods.go:951] [default/openshift-pipelines/tkn-cli-serve-784b566968-5h6jc] addLogicalPort annotation time took 8.552317ms I0428 15:48:24.778800 3786 pods.go:271] [openshift-pipelines/tkn-cli-serve-784b566968-5h6jc] addLogicalPort took 10.034605ms, libovsdb time 947.349µs I0428 15:48:25.133137 3786 cni.go:452] [openshift-pipelines/tkn-cli-serve-784b566968-5h6jc ea07095b8b4fa5cab702a187dc39c60e8999a8360d1978ba5dd4cb31b2926917 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tkn-cli-serve-784b566968-5h6jc ea07095b8b4fa5cab702a187dc39c60e8999a8360d1978ba5dd4cb31b2926917 network default NAD default NAD key ] I0428 15:48:25.142956 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tkn-cli-serve-784b566968-5h6jc, hostIfaceName: ea07095b8b4fa5c, network: default, NAD default, SandboxID: "ea07095b8b4fa5cab702a187dc39c60e8999a8360d1978ba5dd4cb31b2926917", PCI device ID: , UID: "2d043b21-faa7-42a8-a4b0-09a6b73f1725", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0428 15:48:25.201256 3786 cni.go:473] [openshift-pipelines/tkn-cli-serve-784b566968-5h6jc ea07095b8b4fa5cab702a187dc39c60e8999a8360d1978ba5dd4cb31b2926917 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tkn-cli-serve-784b566968-5h6jc ea07095b8b4fa5cab702a187dc39c60e8999a8360d1978ba5dd4cb31b2926917 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:a9:cc:5a:97:bf\",\"name\":\"ea07095b8b4fa5c\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c827561f-d041-46a5-8ab8-9bcfff0489d5\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:48:29.233809 3786 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:29.234445 3786 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:29.269636 3786 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:29.269784 3786 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:29.275577 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0428 15:48:29.275615 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0428 15:48:31.534580 3786 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-86d7b84f9f-9lbdm] creating logical port tsf-tas_fulcio-server-86d7b84f9f-9lbdm for pod on switch ip-10-0-2-7.ec2.internal I0428 15:48:31.534682 3786 kube.go:256] Updating pod tsf-tas/fulcio-server-86d7b84f9f-9lbdm I0428 15:48:31.549052 3786 pod.go:62] [tsf-tas/fulcio-server-86d7b84f9f-9lbdm] pod update took 14.39997ms I0428 15:48:31.549080 3786 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-86d7b84f9f-9lbdm] addLogicalPort annotation time took 14.429206ms I0428 15:48:31.550820 3786 pods.go:271] [tsf-tas/fulcio-server-86d7b84f9f-9lbdm] addLogicalPort took 16.257851ms, libovsdb time 1.188446ms I0428 15:48:31.982420 3786 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-2ngdk-wk5k9] creating logical port tsf-tas_rekor-createtree-job-2ngdk-wk5k9 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:48:31.982579 3786 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-2ngdk-wk5k9 I0428 15:48:32.003239 3786 pod.go:62] [tsf-tas/rekor-createtree-job-2ngdk-wk5k9] pod update took 20.676929ms I0428 15:48:32.003271 3786 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-2ngdk-wk5k9] addLogicalPort annotation time took 20.719428ms I0428 15:48:32.004823 3786 pods.go:271] [tsf-tas/rekor-createtree-job-2ngdk-wk5k9] addLogicalPort took 22.42424ms, libovsdb time 1.025557ms I0428 15:48:32.679204 3786 cni.go:452] [tsf-tas/rekor-createtree-job-2ngdk-wk5k9 0c7e700eeeade0a75bd53058a7de751bb4f36c56998166fdcefe1e031550fdf4 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-2ngdk-wk5k9 0c7e700eeeade0a75bd53058a7de751bb4f36c56998166fdcefe1e031550fdf4 network default NAD default NAD key ] I0428 15:48:32.680144 3786 cni.go:452] [tsf-tas/fulcio-server-86d7b84f9f-9lbdm bceca2acabb4f3161aa532510c99eaacb55c9cb5c716144ef8492ae0a69f696c network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-86d7b84f9f-9lbdm bceca2acabb4f3161aa532510c99eaacb55c9cb5c716144ef8492ae0a69f696c network default NAD default NAD key ] I0428 15:48:32.690038 3786 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-2ngdk-wk5k9, hostIfaceName: 0c7e700eeeade0a, network: default, NAD default, SandboxID: "0c7e700eeeade0a75bd53058a7de751bb4f36c56998166fdcefe1e031550fdf4", PCI device ID: , UID: "4a6a407d-6d67-488b-a942-023a3fc3d1d5", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0428 15:48:32.690743 3786 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-86d7b84f9f-9lbdm, hostIfaceName: bceca2acabb4f31, network: default, NAD default, SandboxID: "bceca2acabb4f3161aa532510c99eaacb55c9cb5c716144ef8492ae0a69f696c", PCI device ID: , UID: "d5acfc52-ba02-4075-9a05-a24364c67d0e", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0428 15:48:32.746810 3786 cni.go:473] [tsf-tas/fulcio-server-86d7b84f9f-9lbdm bceca2acabb4f3161aa532510c99eaacb55c9cb5c716144ef8492ae0a69f696c network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-86d7b84f9f-9lbdm bceca2acabb4f3161aa532510c99eaacb55c9cb5c716144ef8492ae0a69f696c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:f3:e1:ba:be:a0\",\"name\":\"bceca2acabb4f31\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/01a389c4-4b76-4fbe-a5f3-191f4ad48e03\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:48:32.772870 3786 cni.go:473] [tsf-tas/rekor-createtree-job-2ngdk-wk5k9 0c7e700eeeade0a75bd53058a7de751bb4f36c56998166fdcefe1e031550fdf4 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-2ngdk-wk5k9 0c7e700eeeade0a75bd53058a7de751bb4f36c56998166fdcefe1e031550fdf4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:a4:fd:6f:c5:2d\",\"name\":\"0c7e700eeeade0a\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cb702671-8c7a-46ff-9d84-85f4c3189b59\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:48:34.625294 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0428 15:48:34.625325 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0428 15:48:34.653359 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0428 15:48:34.653385 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server I0428 15:48:35.189749 3786 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0428 15:48:38.295016 3786 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-m7t22 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:38.295546 3786 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-m7t22 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:39.175219 3786 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-m7t22 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:39.175244 3786 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-m7t22 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:48:42.178608 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0428 15:48:44.194709 3786 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0428 15:48:44.974672 3786 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0428 15:48:47.989587 3786 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0428 15:48:49.149700 3786 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:49.150292 3786 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:50.032275 3786 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:50.032300 3786 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:56.285039 3786 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-l2x77] creating logical port openshift-pipelines_pipelines-console-plugin-65f788bc74-l2x77 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:48:56.285171 3786 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-65f788bc74-l2x77 I0428 15:48:56.298774 3786 pod.go:62] [openshift-pipelines/pipelines-console-plugin-65f788bc74-l2x77] pod update took 13.631867ms I0428 15:48:56.298799 3786 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-console-plugin-65f788bc74-l2x77] addLogicalPort annotation time took 13.661862ms I0428 15:48:56.300248 3786 pods.go:271] [openshift-pipelines/pipelines-console-plugin-65f788bc74-l2x77] addLogicalPort took 15.222806ms, libovsdb time 1.026709ms I0428 15:48:56.648135 3786 cni.go:452] [openshift-pipelines/pipelines-console-plugin-65f788bc74-l2x77 fdc202faa7b64c6bcfa4c8218d730bd73eaa13710aa149c8755fccd2c6ccbb25 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-l2x77 fdc202faa7b64c6bcfa4c8218d730bd73eaa13710aa149c8755fccd2c6ccbb25 network default NAD default NAD key ] I0428 15:48:56.666719 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-65f788bc74-l2x77, hostIfaceName: fdc202faa7b64c6, network: default, NAD default, SandboxID: "fdc202faa7b64c6bcfa4c8218d730bd73eaa13710aa149c8755fccd2c6ccbb25", PCI device ID: , UID: "b929c576-961a-44a0-b4a0-81439da9976d", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0428 15:48:56.726008 3786 cni.go:473] [openshift-pipelines/pipelines-console-plugin-65f788bc74-l2x77 fdc202faa7b64c6bcfa4c8218d730bd73eaa13710aa149c8755fccd2c6ccbb25 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-65f788bc74-l2x77 fdc202faa7b64c6bcfa4c8218d730bd73eaa13710aa149c8755fccd2c6ccbb25 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:c5:99:ca:f5:44\",\"name\":\"fdc202faa7b64c6\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e5857a2c-4d7b-4e3b-b4ed-18134bb13807\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:49:19.529525 3786 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:19.530124 3786 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:19.548576 3786 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:19.548598 3786 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:19.553935 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0428 15:49:19.553958 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0428 15:49:27.892338 3786 cni.go:452] [tsf-tas/rekor-createtree-job-2ngdk-wk5k9 0c7e700eeeade0a75bd53058a7de751bb4f36c56998166fdcefe1e031550fdf4 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-2ngdk-wk5k9 0c7e700eeeade0a75bd53058a7de751bb4f36c56998166fdcefe1e031550fdf4 network default NAD default NAD key ] I0428 15:49:27.965736 3786 cni.go:473] [tsf-tas/rekor-createtree-job-2ngdk-wk5k9 0c7e700eeeade0a75bd53058a7de751bb4f36c56998166fdcefe1e031550fdf4 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-2ngdk-wk5k9 0c7e700eeeade0a75bd53058a7de751bb4f36c56998166fdcefe1e031550fdf4 network default NAD default NAD key default], result "{}", err I0428 15:49:28.019521 3786 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.019557 3786 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-2ngdk-wk5k9 I0428 15:49:28.021007 3786 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-2ngdk-wk5k9, ips: 10.128.0.43 I0428 15:49:28.021033 3786 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.348391 3786 obj_retry.go:498] Detected object tsf/patch-tekton-config-2rx2d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.349014 3786 obj_retry.go:498] Detected object tsf/patch-tekton-config-2rx2d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.905866 3786 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:28.905894 3786 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:29.377345 3786 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:29.377366 3786 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:32.992440 3786 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-5fdc6f8b87-6jzfj] creating logical port tsf-tas_rekor-search-ui-5fdc6f8b87-6jzfj for pod on switch ip-10-0-2-7.ec2.internal I0428 15:49:32.992530 3786 kube.go:256] Updating pod tsf-tas/rekor-search-ui-5fdc6f8b87-6jzfj I0428 15:49:33.005931 3786 pod.go:62] [tsf-tas/rekor-search-ui-5fdc6f8b87-6jzfj] pod update took 13.425116ms I0428 15:49:33.005953 3786 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-5fdc6f8b87-6jzfj] addLogicalPort annotation time took 13.448527ms I0428 15:49:33.007332 3786 pods.go:271] [tsf-tas/rekor-search-ui-5fdc6f8b87-6jzfj] addLogicalPort took 14.90652ms, libovsdb time 972.957µs I0428 15:49:33.374737 3786 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:33.375307 3786 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:33.567536 3786 cni.go:452] [tsf-tas/rekor-search-ui-5fdc6f8b87-6jzfj 02d01858056f06c991e11ab19d891c8a8817f7e5fdad553d206e1249dc9c314a network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-5fdc6f8b87-6jzfj 02d01858056f06c991e11ab19d891c8a8817f7e5fdad553d206e1249dc9c314a network default NAD default NAD key ] I0428 15:49:33.577283 3786 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-5fdc6f8b87-6jzfj, hostIfaceName: 02d01858056f06c, network: default, NAD default, SandboxID: "02d01858056f06c991e11ab19d891c8a8817f7e5fdad553d206e1249dc9c314a", PCI device ID: , UID: "9937e40d-e83a-4ef6-b455-05c59dbb0e34", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0428 15:49:33.632943 3786 cni.go:473] [tsf-tas/rekor-search-ui-5fdc6f8b87-6jzfj 02d01858056f06c991e11ab19d891c8a8817f7e5fdad553d206e1249dc9c314a network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-5fdc6f8b87-6jzfj 02d01858056f06c991e11ab19d891c8a8817f7e5fdad553d206e1249dc9c314a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:51:99:4a:ff:63\",\"name\":\"02d01858056f06c\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/020e65d2-15cd-4de1-89d4-e149144b59d0\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:49:34.249618 3786 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:34.249644 3786 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:49:36.258971 3786 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0428 15:49:36.947936 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0428 15:49:39.514648 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0428 15:49:40.433522 3786 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:40.434076 3786 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:41.453982 3786 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.454007 3786 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.486167 3786 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.486187 3786 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.490344 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-2rx2d I0428 15:49:41.490376 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-2rx2d W0428 15:49:41.527239 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0428 15:49:45.690554 3786 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:45.691164 3786 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:45.708520 3786 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:45.708541 3786 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:45.713399 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0428 15:49:45.713421 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa W0428 15:49:49.113855 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0428 15:49:51.500866 3786 obj_retry.go:498] Detected object tsf-tpa/create-db-txz9b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:51.501424 3786 obj_retry.go:498] Detected object tsf-tpa/create-db-txz9b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:52.525724 3786 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:52.525751 3786 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:52.635762 3786 base_network_controller_pods.go:487] [default/tsf-tpa/migrate-db-cfwnr] creating logical port tsf-tpa_migrate-db-cfwnr for pod on switch ip-10-0-2-7.ec2.internal I0428 15:49:52.635870 3786 kube.go:256] Updating pod tsf-tpa/migrate-db-cfwnr I0428 15:49:52.648202 3786 pod.go:62] [tsf-tpa/migrate-db-cfwnr] pod update took 12.361743ms I0428 15:49:52.648227 3786 base_network_controller_pods.go:951] [default/tsf-tpa/migrate-db-cfwnr] addLogicalPort annotation time took 12.393206ms I0428 15:49:52.649692 3786 pods.go:271] [tsf-tpa/migrate-db-cfwnr] addLogicalPort took 13.950043ms, libovsdb time 1.006607ms I0428 15:49:52.994918 3786 cni.go:452] [tsf-tpa/migrate-db-cfwnr 7f85bbe2d37cde6bbbc4d997277b1246a09a2383336c30d6a07c97eff7bf586c network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/migrate-db-cfwnr 7f85bbe2d37cde6bbbc4d997277b1246a09a2383336c30d6a07c97eff7bf586c network default NAD default NAD key ] I0428 15:49:53.004218 3786 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: migrate-db-cfwnr, hostIfaceName: 7f85bbe2d37cde6, network: default, NAD default, SandboxID: "7f85bbe2d37cde6bbbc4d997277b1246a09a2383336c30d6a07c97eff7bf586c", PCI device ID: , UID: "412eaf46-d90f-4240-b320-84c431bd80bd", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0428 15:49:53.058818 3786 cni.go:473] [tsf-tpa/migrate-db-cfwnr 7f85bbe2d37cde6bbbc4d997277b1246a09a2383336c30d6a07c97eff7bf586c network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/migrate-db-cfwnr 7f85bbe2d37cde6bbbc4d997277b1246a09a2383336c30d6a07c97eff7bf586c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:9c:89:23:4a:36\",\"name\":\"7f85bbe2d37cde6\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9aac8d92-3d1f-4703-b987-bddc01865caf\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:49:57.048260 3786 cni.go:452] [tsf-tpa/migrate-db-cfwnr 7f85bbe2d37cde6bbbc4d997277b1246a09a2383336c30d6a07c97eff7bf586c network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/migrate-db-cfwnr 7f85bbe2d37cde6bbbc4d997277b1246a09a2383336c30d6a07c97eff7bf586c network default NAD default NAD key ] I0428 15:49:57.141735 3786 cni.go:473] [tsf-tpa/migrate-db-cfwnr 7f85bbe2d37cde6bbbc4d997277b1246a09a2383336c30d6a07c97eff7bf586c network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/migrate-db-cfwnr 7f85bbe2d37cde6bbbc4d997277b1246a09a2383336c30d6a07c97eff7bf586c network default NAD default NAD key default], result "{}", err I0428 15:49:57.190796 3786 obj_retry.go:498] Detected object tsf-tpa/migrate-db-cfwnr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:57.190826 3786 pods.go:176] Deleting pod: tsf-tpa/migrate-db-cfwnr I0428 15:49:57.192084 3786 pods.go:236] Attempting to release IPs for pod: tsf-tpa/migrate-db-cfwnr, ips: 10.128.0.46 I0428 15:49:57.192133 3786 obj_retry.go:498] Detected object tsf-tpa/migrate-db-cfwnr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:58.214447 3786 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:58.214471 3786 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:58.414499 3786 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-cgccv] creating logical port tsf-tpa_create-importers-cgccv for pod on switch ip-10-0-2-7.ec2.internal I0428 15:49:58.414589 3786 kube.go:256] Updating pod tsf-tpa/create-importers-cgccv I0428 15:49:58.427205 3786 pod.go:62] [tsf-tpa/create-importers-cgccv] pod update took 12.637373ms I0428 15:49:58.427229 3786 base_network_controller_pods.go:951] [default/tsf-tpa/create-importers-cgccv] addLogicalPort annotation time took 12.669081ms I0428 15:49:58.428532 3786 pods.go:271] [tsf-tpa/create-importers-cgccv] addLogicalPort took 14.0505ms, libovsdb time 873.325µs I0428 15:49:58.774868 3786 cni.go:452] [tsf-tpa/create-importers-cgccv 5d1787ed3f9996754c6d5789a9f753c3c9f7ff64b9bff303e5c6fd7ab980d0d4 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-importers-cgccv 5d1787ed3f9996754c6d5789a9f753c3c9f7ff64b9bff303e5c6fd7ab980d0d4 network default NAD default NAD key ] I0428 15:49:58.784904 3786 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-cgccv, hostIfaceName: 5d1787ed3f99967, network: default, NAD default, SandboxID: "5d1787ed3f9996754c6d5789a9f753c3c9f7ff64b9bff303e5c6fd7ab980d0d4", PCI device ID: , UID: "523d1948-43d0-4c5d-9a45-07f645642f93", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0428 15:49:58.838539 3786 cni.go:473] [tsf-tpa/create-importers-cgccv 5d1787ed3f9996754c6d5789a9f753c3c9f7ff64b9bff303e5c6fd7ab980d0d4 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-importers-cgccv 5d1787ed3f9996754c6d5789a9f753c3c9f7ff64b9bff303e5c6fd7ab980d0d4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:1e:41:18:e1:d6\",\"name\":\"5d1787ed3f99967\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7bebe764-37dc-4864-956c-ad8b5c19003d\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:50:00.068903 3786 cni.go:452] [tsf-tpa/create-importers-cgccv 5d1787ed3f9996754c6d5789a9f753c3c9f7ff64b9bff303e5c6fd7ab980d0d4 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-importers-cgccv 5d1787ed3f9996754c6d5789a9f753c3c9f7ff64b9bff303e5c6fd7ab980d0d4 network default NAD default NAD key ] I0428 15:50:00.161966 3786 cni.go:473] [tsf-tpa/create-importers-cgccv 5d1787ed3f9996754c6d5789a9f753c3c9f7ff64b9bff303e5c6fd7ab980d0d4 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-importers-cgccv 5d1787ed3f9996754c6d5789a9f753c3c9f7ff64b9bff303e5c6fd7ab980d0d4 network default NAD default NAD key default], result "{}", err I0428 15:50:00.172252 3786 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29623190-qm4qv for pod on switch ip-10-0-2-7.ec2.internal I0428 15:50:00.172362 3786 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv I0428 15:50:00.183212 3786 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv] pod update took 10.881806ms I0428 15:50:00.183233 3786 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv] addLogicalPort annotation time took 10.909136ms I0428 15:50:00.184517 3786 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv] addLogicalPort took 12.283007ms, libovsdb time 872.296µs I0428 15:50:00.218328 3786 obj_retry.go:498] Detected object tsf-tpa/create-importers-cgccv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:00.218381 3786 pods.go:176] Deleting pod: tsf-tpa/create-importers-cgccv I0428 15:50:00.219601 3786 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-cgccv, ips: 10.128.0.47 I0428 15:50:00.219627 3786 obj_retry.go:498] Detected object tsf-tpa/create-importers-cgccv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:00.542822 3786 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv aef218e0dd1a0c2d17da836c2e148a4dd698542a1af35a1ed23b41cc0ec28194 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv aef218e0dd1a0c2d17da836c2e148a4dd698542a1af35a1ed23b41cc0ec28194 network default NAD default NAD key ] I0428 15:50:00.552691 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29623190-qm4qv, hostIfaceName: aef218e0dd1a0c2, network: default, NAD default, SandboxID: "aef218e0dd1a0c2d17da836c2e148a4dd698542a1af35a1ed23b41cc0ec28194", PCI device ID: , UID: "559d9498-bbb4-4d47-a467-762bd17c69c2", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0428 15:50:00.606458 3786 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv aef218e0dd1a0c2d17da836c2e148a4dd698542a1af35a1ed23b41cc0ec28194 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv aef218e0dd1a0c2d17da836c2e148a4dd698542a1af35a1ed23b41cc0ec28194 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:18:28:74:cb:00\",\"name\":\"aef218e0dd1a0c2\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b8b602cf-aba0-4983-8041-8168761ce67c\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:50:01.248146 3786 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:01.248174 3786 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:04.094768 3786 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv aef218e0dd1a0c2d17da836c2e148a4dd698542a1af35a1ed23b41cc0ec28194 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv aef218e0dd1a0c2d17da836c2e148a4dd698542a1af35a1ed23b41cc0ec28194 network default NAD default NAD key ] I0428 15:50:04.184817 3786 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv aef218e0dd1a0c2d17da836c2e148a4dd698542a1af35a1ed23b41cc0ec28194 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv aef218e0dd1a0c2d17da836c2e148a4dd698542a1af35a1ed23b41cc0ec28194 network default NAD default NAD key default], result "{}", err I0428 15:50:04.234312 3786 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:04.234344 3786 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv I0428 15:50:04.235612 3786 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv, ips: 10.128.0.48 I0428 15:50:04.235647 3786 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:05.254661 3786 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:05.254688 3786 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:07.191779 3786 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 W0428 15:50:08.447677 3786 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0428 15:50:11.608828 3786 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:11.609418 3786 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:12.639339 3786 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:12.639366 3786 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:50:28.561078 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0428 15:50:29.956690 3786 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:29.957311 3786 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:29.995084 3786 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:29.995130 3786 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:30.000629 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0428 15:50:30.000657 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0428 15:50:30.708210 3786 namespace.go:98] [image-controller] adding namespace I0428 15:50:30.709202 3786 namespace.go:102] [image-controller] adding namespace took 960.864µs I0428 15:50:30.713434 3786 namespace.go:98] [build-service] adding namespace I0428 15:50:30.714262 3786 namespace.go:102] [build-service] adding namespace took 813.224µs I0428 15:50:30.714270 3786 namespace.go:98] [integration-service] adding namespace I0428 15:50:30.714871 3786 namespace.go:102] [integration-service] adding namespace took 572.272µs I0428 15:50:30.727200 3786 namespace.go:98] [default-tenant] adding namespace I0428 15:50:30.727951 3786 namespace.go:102] [default-tenant] adding namespace took 733.143µs I0428 15:50:30.767566 3786 namespace.go:142] [image-controller] updating namespace I0428 15:50:30.798943 3786 namespace.go:142] [image-controller] updating namespace I0428 15:50:30.806923 3786 namespace.go:142] [build-service] updating namespace I0428 15:50:30.833570 3786 namespace.go:142] [build-service] updating namespace I0428 15:50:30.841459 3786 namespace.go:142] [integration-service] updating namespace I0428 15:50:30.865448 3786 namespace.go:142] [integration-service] updating namespace I0428 15:50:30.879470 3786 namespace.go:142] [default-tenant] updating namespace I0428 15:50:30.895320 3786 namespace.go:142] [default-tenant] updating namespace I0428 15:50:31.003063 3786 namespace.go:142] [build-service] updating namespace I0428 15:50:31.041195 3786 namespace.go:142] [integration-service] updating namespace I0428 15:50:31.106888 3786 namespace.go:98] [release-service] adding namespace I0428 15:50:31.107825 3786 namespace.go:102] [release-service] adding namespace took 908.162µs I0428 15:50:31.152589 3786 namespace.go:142] [konflux-ui] updating namespace I0428 15:50:31.178189 3786 namespace.go:142] [release-service] updating namespace I0428 15:50:31.204428 3786 namespace.go:142] [release-service] updating namespace I0428 15:50:31.210345 3786 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-2-7.ec2.internal I0428 15:50:31.210449 3786 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0428 15:50:31.274613 3786 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0428 15:50:31.311411 3786 namespace.go:98] [konflux-info] adding namespace I0428 15:50:31.312383 3786 namespace.go:102] [konflux-info] adding namespace took 949.463µs I0428 15:50:31.366360 3786 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 155.929565ms I0428 15:50:31.366394 3786 base_network_controller_pods.go:951] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 155.969677ms I0428 15:50:31.368001 3786 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 157.670605ms, libovsdb time 1.062578ms I0428 15:50:31.409366 3786 namespace.go:142] [konflux-info] updating namespace I0428 15:50:31.449496 3786 namespace.go:98] [namespace-lister] adding namespace I0428 15:50:31.450480 3786 namespace.go:102] [namespace-lister] adding namespace took 956.954µs I0428 15:50:31.476358 3786 namespace.go:142] [konflux-info] updating namespace I0428 15:50:31.504263 3786 namespace.go:98] [enterprise-contract-service] adding namespace I0428 15:50:31.505195 3786 namespace.go:102] [enterprise-contract-service] adding namespace took 909.027µs I0428 15:50:31.543395 3786 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:31.544352 3786 cni.go:452] [konflux-ui/test-tsf-konflux db8aa09aa98cd5a96674cf3575ba4994c5bc22ee456445eb95b51cfba906460f network default NAD default NAD key ] ADD starting CNI request [konflux-ui/test-tsf-konflux db8aa09aa98cd5a96674cf3575ba4994c5bc22ee456445eb95b51cfba906460f network default NAD default NAD key ] I0428 15:50:31.571421 3786 namespace.go:142] [image-controller] updating namespace I0428 15:50:31.571466 3786 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: db8aa09aa98cd5a, network: default, NAD default, SandboxID: "db8aa09aa98cd5a96674cf3575ba4994c5bc22ee456445eb95b51cfba906460f", PCI device ID: , UID: "83f82751-eefe-4ae6-a738-117388e3b27c", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0428 15:50:31.631348 3786 cni.go:473] [konflux-ui/test-tsf-konflux db8aa09aa98cd5a96674cf3575ba4994c5bc22ee456445eb95b51cfba906460f network default NAD default NAD key default] ADD finished CNI request [konflux-ui/test-tsf-konflux db8aa09aa98cd5a96674cf3575ba4994c5bc22ee456445eb95b51cfba906460f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:c8:af:58:c2:d9\",\"name\":\"db8aa09aa98cd5a\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/649f5ab3-b434-44ef-a510-154f02ef8b9f\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:50:31.745324 3786 namespace.go:142] [default-tenant] updating namespace I0428 15:50:31.885288 3786 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:32.061334 3786 namespace.go:98] [segment-bridge] adding namespace I0428 15:50:32.062304 3786 namespace.go:102] [segment-bridge] adding namespace took 941.121µs I0428 15:50:32.165588 3786 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:32.165646 3786 namespace.go:98] [konflux-cli] adding namespace I0428 15:50:32.166510 3786 namespace.go:102] [konflux-cli] adding namespace took 846.311µs I0428 15:50:32.318270 3786 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0428 15:50:32.318922 3786 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0428 15:50:32.318944 3786 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0428 15:50:32.322103 3786 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0428 15:50:32.336198 3786 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:32.499479 3786 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0428 15:50:32.499512 3786 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0428 15:50:32.500437 3786 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0428 15:50:32.515040 3786 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:32.685507 3786 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:32.914654 3786 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:32.954720 3786 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:33.029608 3786 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-696b855848-q85vb] creating logical port namespace-lister_namespace-lister-696b855848-q85vb for pod on switch ip-10-0-2-7.ec2.internal I0428 15:50:33.029756 3786 kube.go:256] Updating pod namespace-lister/namespace-lister-696b855848-q85vb I0428 15:50:33.126453 3786 pod.go:62] [namespace-lister/namespace-lister-696b855848-q85vb] pod update took 96.722235ms I0428 15:50:33.126482 3786 base_network_controller_pods.go:951] [default/namespace-lister/namespace-lister-696b855848-q85vb] addLogicalPort annotation time took 96.758654ms I0428 15:50:33.127811 3786 pods.go:271] [namespace-lister/namespace-lister-696b855848-q85vb] addLogicalPort took 98.222592ms, libovsdb time 909.835µs I0428 15:50:33.341040 3786 base_network_controller_pods.go:487] [default/konflux-ui/dex-566898b497-4zgrk] creating logical port konflux-ui_dex-566898b497-4zgrk for pod on switch ip-10-0-2-7.ec2.internal I0428 15:50:33.341193 3786 kube.go:256] Updating pod konflux-ui/dex-566898b497-4zgrk I0428 15:50:33.350548 3786 base_network_controller_pods.go:487] [default/konflux-ui/proxy-c8b546c9d-qbnsp] creating logical port konflux-ui_proxy-c8b546c9d-qbnsp for pod on switch ip-10-0-2-7.ec2.internal I0428 15:50:33.350696 3786 kube.go:256] Updating pod konflux-ui/proxy-c8b546c9d-qbnsp I0428 15:50:33.386064 3786 namespace.go:142] [build-service] updating namespace I0428 15:50:33.398384 3786 pod.go:62] [konflux-ui/dex-566898b497-4zgrk] pod update took 57.21745ms I0428 15:50:33.398410 3786 base_network_controller_pods.go:951] [default/konflux-ui/dex-566898b497-4zgrk] addLogicalPort annotation time took 57.249693ms I0428 15:50:33.399809 3786 pods.go:271] [konflux-ui/dex-566898b497-4zgrk] addLogicalPort took 58.79111ms, libovsdb time 961.33µs I0428 15:50:33.437647 3786 pod.go:62] [konflux-ui/proxy-c8b546c9d-qbnsp] pod update took 86.984737ms I0428 15:50:33.437679 3786 base_network_controller_pods.go:951] [default/konflux-ui/proxy-c8b546c9d-qbnsp] addLogicalPort annotation time took 87.023983ms I0428 15:50:33.439213 3786 pods.go:271] [konflux-ui/proxy-c8b546c9d-qbnsp] addLogicalPort took 88.690723ms, libovsdb time 968.944µs I0428 15:50:33.757363 3786 namespace.go:142] [default-tenant] updating namespace I0428 15:50:33.886895 3786 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p] creating logical port integration-service_integration-service-controller-manager-7b666bf6d8-5qf9p for pod on switch ip-10-0-2-7.ec2.internal I0428 15:50:33.887011 3786 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p I0428 15:50:33.930350 3786 pod.go:62] [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p] pod update took 43.372843ms I0428 15:50:33.930384 3786 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p] addLogicalPort annotation time took 43.411991ms I0428 15:50:33.931748 3786 pods.go:271] [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p] addLogicalPort took 44.879205ms, libovsdb time 880.545µs I0428 15:50:34.002742 3786 cni.go:452] [namespace-lister/namespace-lister-696b855848-q85vb b137def8693f0cbdef721d77c7f49d6059f95dd51b037d50b71c6e0f6ee4c7c5 network default NAD default NAD key ] ADD starting CNI request [namespace-lister/namespace-lister-696b855848-q85vb b137def8693f0cbdef721d77c7f49d6059f95dd51b037d50b71c6e0f6ee4c7c5 network default NAD default NAD key ] I0428 15:50:34.012298 3786 helper_linux.go:556] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-696b855848-q85vb, hostIfaceName: b137def8693f0cb, network: default, NAD default, SandboxID: "b137def8693f0cbdef721d77c7f49d6059f95dd51b037d50b71c6e0f6ee4c7c5", PCI device ID: , UID: "450ce0a8-a652-4396-89bf-97247f275155", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0428 15:50:34.067592 3786 cni.go:473] [namespace-lister/namespace-lister-696b855848-q85vb b137def8693f0cbdef721d77c7f49d6059f95dd51b037d50b71c6e0f6ee4c7c5 network default NAD default NAD key default] ADD finished CNI request [namespace-lister/namespace-lister-696b855848-q85vb b137def8693f0cbdef721d77c7f49d6059f95dd51b037d50b71c6e0f6ee4c7c5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:85:92:f9:ad:e3\",\"name\":\"b137def8693f0cb\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/00cf8232-9e10-463e-af88-f6424860b58b\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:50:34.081218 3786 namespace.go:142] [image-controller] updating namespace I0428 15:50:34.344183 3786 namespace.go:142] [integration-service] updating namespace I0428 15:50:34.439572 3786 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7cbf488fcf-m4vbd] creating logical port build-service_build-service-controller-manager-7cbf488fcf-m4vbd for pod on switch ip-10-0-2-7.ec2.internal I0428 15:50:34.439682 3786 kube.go:256] Updating pod build-service/build-service-controller-manager-7cbf488fcf-m4vbd I0428 15:50:34.464158 3786 pod.go:62] [build-service/build-service-controller-manager-7cbf488fcf-m4vbd] pod update took 24.506916ms I0428 15:50:34.464184 3786 base_network_controller_pods.go:951] [default/build-service/build-service-controller-manager-7cbf488fcf-m4vbd] addLogicalPort annotation time took 24.539466ms I0428 15:50:34.465483 3786 pods.go:271] [build-service/build-service-controller-manager-7cbf488fcf-m4vbd] addLogicalPort took 25.933998ms, libovsdb time 812.387µs I0428 15:50:34.561430 3786 namespace.go:142] [release-service] updating namespace I0428 15:50:34.810499 3786 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7] creating logical port release-service_release-service-controller-manager-6bfdb6bcb6-pnzg7 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:50:34.810672 3786 kube.go:256] Updating pod release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7 I0428 15:50:34.821222 3786 cni.go:452] [build-service/build-service-controller-manager-7cbf488fcf-m4vbd 08d2b69ee5739c7ea201b01669994e5ec11648abedd9ff71b8274dcbd8a5bdcf network default NAD default NAD key ] ADD starting CNI request [build-service/build-service-controller-manager-7cbf488fcf-m4vbd 08d2b69ee5739c7ea201b01669994e5ec11648abedd9ff71b8274dcbd8a5bdcf network default NAD default NAD key ] I0428 15:50:34.833256 3786 helper_linux.go:556] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7cbf488fcf-m4vbd, hostIfaceName: 08d2b69ee5739c7, network: default, NAD default, SandboxID: "08d2b69ee5739c7ea201b01669994e5ec11648abedd9ff71b8274dcbd8a5bdcf", PCI device ID: , UID: "e3402e33-1bd3-489e-a439-c2be67380f23", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0428 15:50:34.851372 3786 namespace.go:142] [build-service] updating namespace I0428 15:50:34.865296 3786 kube.go:256] Updating pod release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7 I0428 15:50:34.906901 3786 cni.go:473] [build-service/build-service-controller-manager-7cbf488fcf-m4vbd 08d2b69ee5739c7ea201b01669994e5ec11648abedd9ff71b8274dcbd8a5bdcf network default NAD default NAD key default] ADD finished CNI request [build-service/build-service-controller-manager-7cbf488fcf-m4vbd 08d2b69ee5739c7ea201b01669994e5ec11648abedd9ff71b8274dcbd8a5bdcf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:59:da:2f:ce:eb\",\"name\":\"08d2b69ee5739c7\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c9aa8068-b836-4949-b15e-121af496e199\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:50:35.022443 3786 namespace.go:142] [default-tenant] updating namespace I0428 15:50:35.030640 3786 pod.go:62] [release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7] pod update took 220.007508ms I0428 15:50:35.030673 3786 base_network_controller_pods.go:951] [default/release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7] addLogicalPort annotation time took 220.047283ms I0428 15:50:35.032223 3786 pods.go:271] [release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7] addLogicalPort took 221.743516ms, libovsdb time 924.941µs I0428 15:50:35.124894 3786 namespace.go:142] [image-controller] updating namespace I0428 15:50:35.196339 3786 namespace.go:142] [integration-service] updating namespace I0428 15:50:35.256311 3786 namespace.go:142] [release-service] updating namespace I0428 15:50:35.448335 3786 namespace.go:142] [build-service] updating namespace I0428 15:50:35.563604 3786 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-6dd7d45f78-wpzl6] creating logical port integration-service_integration-service-controller-manager-6dd7d45f78-wpzl6 for pod on switch ip-10-0-2-7.ec2.internal I0428 15:50:35.563708 3786 kube.go:256] Updating pod integration-service/integration-service-controller-manager-6dd7d45f78-wpzl6 I0428 15:50:35.661062 3786 pod.go:62] [integration-service/integration-service-controller-manager-6dd7d45f78-wpzl6] pod update took 97.378798ms I0428 15:50:35.661121 3786 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-6dd7d45f78-wpzl6] addLogicalPort annotation time took 97.414137ms I0428 15:50:35.662788 3786 pods.go:271] [integration-service/integration-service-controller-manager-6dd7d45f78-wpzl6] addLogicalPort took 99.206991ms, libovsdb time 1.074333ms I0428 15:50:35.728198 3786 namespace.go:142] [default-tenant] updating namespace I0428 15:50:35.844330 3786 namespace.go:142] [image-controller] updating namespace I0428 15:50:35.924599 3786 namespace.go:142] [integration-service] updating namespace I0428 15:50:35.989254 3786 namespace.go:142] [release-service] updating namespace W0428 15:50:36.878491 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0428 15:50:37.237630 3786 namespace.go:142] [enterprise-contract-service] updating namespace W0428 15:50:37.297653 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0428 15:50:37.335449 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0428 15:50:37.437080 3786 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:37.636868 3786 namespace.go:142] [konflux-info] updating namespace I0428 15:50:37.839698 3786 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:38.039189 3786 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:38.159131 3786 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:38.172823 3786 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:38.188833 3786 namespace.go:142] [konflux-info] updating namespace I0428 15:50:38.205638 3786 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:38.220659 3786 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:38.266951 3786 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:38.372272 3786 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:38.465218 3786 namespace.go:142] [konflux-info] updating namespace I0428 15:50:38.567177 3786 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:38.663732 3786 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:41.069975 3786 cni.go:452] [integration-service/integration-service-controller-manager-6dd7d45f78-wpzl6 4707a593afc85c56bd20df573d492658d8aed38ffc13d03bef87ec21f1d01d68 network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-6dd7d45f78-wpzl6 4707a593afc85c56bd20df573d492658d8aed38ffc13d03bef87ec21f1d01d68 network default NAD default NAD key ] I0428 15:50:41.080282 3786 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-6dd7d45f78-wpzl6, hostIfaceName: 4707a593afc85c5, network: default, NAD default, SandboxID: "4707a593afc85c56bd20df573d492658d8aed38ffc13d03bef87ec21f1d01d68", PCI device ID: , UID: "e8614ad2-2c0e-4770-acff-aa41aea64f11", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0428 15:50:41.139015 3786 cni.go:473] [integration-service/integration-service-controller-manager-6dd7d45f78-wpzl6 4707a593afc85c56bd20df573d492658d8aed38ffc13d03bef87ec21f1d01d68 network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-6dd7d45f78-wpzl6 4707a593afc85c56bd20df573d492658d8aed38ffc13d03bef87ec21f1d01d68 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:51:4c:67:94:3d\",\"name\":\"4707a593afc85c5\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe450666-bc1c-4413-ab89-c9913ff5b8a1\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:50:41.209461 3786 cni.go:452] [konflux-ui/dex-566898b497-4zgrk b153ae4148696fed06440f4a792c94a684290dcfbe7862ec2d10f2223aa89bfc network default NAD default NAD key ] ADD starting CNI request [konflux-ui/dex-566898b497-4zgrk b153ae4148696fed06440f4a792c94a684290dcfbe7862ec2d10f2223aa89bfc network default NAD default NAD key ] I0428 15:50:41.214317 3786 cni.go:452] [konflux-ui/proxy-c8b546c9d-qbnsp c30e29ca5103a9ea11f2b0ff428314f6a2d2482439c3777db1cfeb88689d8af0 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-c8b546c9d-qbnsp c30e29ca5103a9ea11f2b0ff428314f6a2d2482439c3777db1cfeb88689d8af0 network default NAD default NAD key ] I0428 15:50:41.221040 3786 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: dex-566898b497-4zgrk, hostIfaceName: b153ae4148696fe, network: default, NAD default, SandboxID: "b153ae4148696fed06440f4a792c94a684290dcfbe7862ec2d10f2223aa89bfc", PCI device ID: , UID: "29f4dddc-4401-4eb4-9ae9-466ca444331c", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0428 15:50:41.224978 3786 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-c8b546c9d-qbnsp, hostIfaceName: c30e29ca5103a9e, network: default, NAD default, SandboxID: "c30e29ca5103a9ea11f2b0ff428314f6a2d2482439c3777db1cfeb88689d8af0", PCI device ID: , UID: "33ee126f-a503-43dd-ad1b-d23fbc779e5f", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0428 15:50:41.276645 3786 cni.go:473] [konflux-ui/dex-566898b497-4zgrk b153ae4148696fed06440f4a792c94a684290dcfbe7862ec2d10f2223aa89bfc network default NAD default NAD key default] ADD finished CNI request [konflux-ui/dex-566898b497-4zgrk b153ae4148696fed06440f4a792c94a684290dcfbe7862ec2d10f2223aa89bfc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:23:97:22:93:c9\",\"name\":\"b153ae4148696fe\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5094e5ef-f65e-420d-8d69-ca089c7cc004\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:50:41.297119 3786 cni.go:473] [konflux-ui/proxy-c8b546c9d-qbnsp c30e29ca5103a9ea11f2b0ff428314f6a2d2482439c3777db1cfeb88689d8af0 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-c8b546c9d-qbnsp c30e29ca5103a9ea11f2b0ff428314f6a2d2482439c3777db1cfeb88689d8af0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:e4:cc:a9:76:5c\",\"name\":\"c30e29ca5103a9e\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b6b47f70-0b9e-41b6-b9e1-525ecaade284\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:50:41.353976 3786 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0428 15:50:41.735195 3786 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p d1012a129a9e8f64300244d0d606a7bb842230ddedd8a72cc4fee8dee4723091 network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p d1012a129a9e8f64300244d0d606a7bb842230ddedd8a72cc4fee8dee4723091 network default NAD default NAD key ] I0428 15:50:41.745295 3786 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-7b666bf6d8-5qf9p, hostIfaceName: d1012a129a9e8f6, network: default, NAD default, SandboxID: "d1012a129a9e8f64300244d0d606a7bb842230ddedd8a72cc4fee8dee4723091", PCI device ID: , UID: "526af8ab-fd47-4fab-a193-ee677ddf1d81", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0428 15:50:41.800044 3786 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p d1012a129a9e8f64300244d0d606a7bb842230ddedd8a72cc4fee8dee4723091 network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p d1012a129a9e8f64300244d0d606a7bb842230ddedd8a72cc4fee8dee4723091 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:79:e0:9c:26:7b\",\"name\":\"d1012a129a9e8f6\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/13f8c16b-9a4f-42aa-b4a6-88df0c6e9a77\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 15:50:42.693201 3786 cni.go:452] [release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7 26c69c8b83484cbb722f813a3ddf805a53233cabb0a48deae4ffa908e771efe8 network default NAD default NAD key ] ADD starting CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7 26c69c8b83484cbb722f813a3ddf805a53233cabb0a48deae4ffa908e771efe8 network default NAD default NAD key ] I0428 15:50:42.704678 3786 helper_linux.go:556] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-6bfdb6bcb6-pnzg7, hostIfaceName: 26c69c8b83484cb, network: default, NAD default, SandboxID: "26c69c8b83484cbb722f813a3ddf805a53233cabb0a48deae4ffa908e771efe8", PCI device ID: , UID: "7332af35-ec50-4d3a-914d-58f69f37d890", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0428 15:50:42.775258 3786 cni.go:473] [release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7 26c69c8b83484cbb722f813a3ddf805a53233cabb0a48deae4ffa908e771efe8 network default NAD default NAD key default] ADD finished CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-pnzg7 26c69c8b83484cbb722f813a3ddf805a53233cabb0a48deae4ffa908e771efe8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:7c:5b:d2:16:73\",\"name\":\"26c69c8b83484cb\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90705631-be5b-4ec7-bf33-71db145aada2\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0428 15:50:43.337285 3786 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0428 15:50:47.927165 3786 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:47.927185 3786 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:48.387234 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 15:50:48.388746 3786 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 15:50:48.454318 3786 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:48.454335 3786 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:48.474437 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0428 15:50:48.476249 3786 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0428 15:50:48.498742 3786 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 15:50:48.498762 3786 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 15:50:52.327851 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0428 15:50:59.384379 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0428 15:50:59.404385 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 15:50:59.410122 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 15:50:59.437651 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0428 15:50:59.442958 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service I0428 15:50:59.598556 3786 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p d1012a129a9e8f64300244d0d606a7bb842230ddedd8a72cc4fee8dee4723091 network default NAD default NAD key ] DEL starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p d1012a129a9e8f64300244d0d606a7bb842230ddedd8a72cc4fee8dee4723091 network default NAD default NAD key ] I0428 15:50:59.695485 3786 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p d1012a129a9e8f64300244d0d606a7bb842230ddedd8a72cc4fee8dee4723091 network default NAD default NAD key default] DEL finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p d1012a129a9e8f64300244d0d606a7bb842230ddedd8a72cc4fee8dee4723091 network default NAD default NAD key default], result "{}", err I0428 15:50:59.753404 3786 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:59.753434 3786 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p I0428 15:50:59.754836 3786 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p, ips: 10.128.0.53 I0428 15:50:59.754863 3786 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:51:00.451010 3786 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:00.451031 3786 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:00.460225 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p I0428 15:51:00.460251 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p I0428 15:51:15.215595 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 W0428 15:51:25.443583 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0428 15:51:25.443606 3786 util.go:959] Failed to build node endpoints for node ip-10-0-2-7.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy I0428 15:51:28.099197 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=12 I0428 15:51:33.194806 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=9 I0428 15:51:44.200318 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=11 I0428 15:51:57.311304 3786 cni.go:452] [konflux-ui/proxy-c8b546c9d-qbnsp c30e29ca5103a9ea11f2b0ff428314f6a2d2482439c3777db1cfeb88689d8af0 network default NAD default NAD key ] DEL starting CNI request [konflux-ui/proxy-c8b546c9d-qbnsp c30e29ca5103a9ea11f2b0ff428314f6a2d2482439c3777db1cfeb88689d8af0 network default NAD default NAD key ] I0428 15:51:57.394185 3786 cni.go:473] [konflux-ui/proxy-c8b546c9d-qbnsp c30e29ca5103a9ea11f2b0ff428314f6a2d2482439c3777db1cfeb88689d8af0 network default NAD default NAD key default] DEL finished CNI request [konflux-ui/proxy-c8b546c9d-qbnsp c30e29ca5103a9ea11f2b0ff428314f6a2d2482439c3777db1cfeb88689d8af0 network default NAD default NAD key default], result "{}", err I0428 15:51:57.465715 3786 obj_retry.go:498] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:51:57.465746 3786 pods.go:176] Deleting pod: konflux-ui/proxy-c8b546c9d-qbnsp I0428 15:51:57.467184 3786 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-c8b546c9d-qbnsp, ips: 10.128.0.52 I0428 15:51:57.467213 3786 obj_retry.go:498] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:51:57.764153 3786 obj_retry.go:492] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:57.764186 3786 obj_retry.go:492] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:57.770768 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-c8b546c9d-qbnsp I0428 15:51:57.770796 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-c8b546c9d-qbnsp I0428 15:52:02.211203 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0428 15:52:02.808692 3786 cni.go:452] [konflux-ui/test-tsf-konflux db8aa09aa98cd5a96674cf3575ba4994c5bc22ee456445eb95b51cfba906460f network default NAD default NAD key ] DEL starting CNI request [konflux-ui/test-tsf-konflux db8aa09aa98cd5a96674cf3575ba4994c5bc22ee456445eb95b51cfba906460f network default NAD default NAD key ] I0428 15:52:02.887208 3786 cni.go:473] [konflux-ui/test-tsf-konflux db8aa09aa98cd5a96674cf3575ba4994c5bc22ee456445eb95b51cfba906460f network default NAD default NAD key default] DEL finished CNI request [konflux-ui/test-tsf-konflux db8aa09aa98cd5a96674cf3575ba4994c5bc22ee456445eb95b51cfba906460f network default NAD default NAD key default], result "{}", err I0428 15:52:02.939466 3786 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:52:02.939500 3786 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0428 15:52:02.940864 3786 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.128.0.49 I0428 15:52:02.940897 3786 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:52:02.955213 3786 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:52:02.955235 3786 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:52:02.964956 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0428 15:52:02.964977 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0428 15:52:16.228185 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0428 15:52:17.101810 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=82 I0428 15:52:22.205772 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0428 15:52:32.870979 3786 namespace.go:98] [managed-tenant-v8l0n] adding namespace I0428 15:52:32.871960 3786 namespace.go:102] [managed-tenant-v8l0n] adding namespace took 953.444µs I0428 15:52:32.894118 3786 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:32.916270 3786 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:34.056657 3786 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:34.194186 3786 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:34.295853 3786 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:53:43.222376 3786 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0428 15:53:54.192321 3786 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=7 I0428 15:53:54.228332 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0428 15:54:03.103345 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=156 I0428 15:55:18.110941 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=516 I0428 15:55:43.104272 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=117 I0428 15:56:06.106195 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=76 I0428 15:57:21.218384 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0428 15:57:38.231108 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0428 15:57:54.194570 3786 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0428 15:58:45.214210 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0428 15:59:12.105619 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=7 I0428 15:59:21.104180 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8 I0428 15:59:31.203540 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0428 15:59:35.225623 3786 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0428 15:59:52.045526 3786 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:52.045574 3786 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:52.050039 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-txz9b I0428 15:59:52.050076 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-txz9b I0428 15:59:58.028186 3786 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:58.028244 3786 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:58.030841 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-cfwnr I0428 15:59:58.030865 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-cfwnr I0428 16:00:00.214728 3786 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29623200-9j7ns for pod on switch ip-10-0-2-7.ec2.internal I0428 16:00:00.214833 3786 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns I0428 16:00:00.247277 3786 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns] pod update took 32.464444ms I0428 16:00:00.247304 3786 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns] addLogicalPort annotation time took 32.495933ms I0428 16:00:00.248728 3786 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns] addLogicalPort took 34.012126ms, libovsdb time 1.00476ms I0428 16:00:00.266624 3786 base_network_controller_pods.go:487] [default/segment-bridge/segment-bridge-29623200-2fqjs] creating logical port segment-bridge_segment-bridge-29623200-2fqjs for pod on switch ip-10-0-2-7.ec2.internal I0428 16:00:00.266721 3786 kube.go:256] Updating pod segment-bridge/segment-bridge-29623200-2fqjs I0428 16:00:00.282130 3786 pod.go:62] [segment-bridge/segment-bridge-29623200-2fqjs] pod update took 15.42871ms I0428 16:00:00.282153 3786 base_network_controller_pods.go:951] [default/segment-bridge/segment-bridge-29623200-2fqjs] addLogicalPort annotation time took 15.455756ms I0428 16:00:00.283514 3786 pods.go:271] [segment-bridge/segment-bridge-29623200-2fqjs] addLogicalPort took 16.900598ms, libovsdb time 879.721µs I0428 16:00:00.574845 3786 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns c5c8cbc17977a856517555f870f5be72de7ea1c4e16949b81dcf2521e3f54025 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns c5c8cbc17977a856517555f870f5be72de7ea1c4e16949b81dcf2521e3f54025 network default NAD default NAD key ] I0428 16:00:00.584719 3786 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29623200-9j7ns, hostIfaceName: c5c8cbc17977a85, network: default, NAD default, SandboxID: "c5c8cbc17977a856517555f870f5be72de7ea1c4e16949b81dcf2521e3f54025", PCI device ID: , UID: "1bf98941-e7a4-40a6-80b8-6c2135f0be8d", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0428 16:00:00.629328 3786 cni.go:452] [segment-bridge/segment-bridge-29623200-2fqjs ca2d92dc3fea7c503fc1a735d5c47878d1531ceef6cc641b587737e743c9a1b3 network default NAD default NAD key ] ADD starting CNI request [segment-bridge/segment-bridge-29623200-2fqjs ca2d92dc3fea7c503fc1a735d5c47878d1531ceef6cc641b587737e743c9a1b3 network default NAD default NAD key ] I0428 16:00:00.641179 3786 helper_linux.go:556] ConfigureOVS: namespace: segment-bridge, podName: segment-bridge-29623200-2fqjs, hostIfaceName: ca2d92dc3fea7c5, network: default, NAD default, SandboxID: "ca2d92dc3fea7c503fc1a735d5c47878d1531ceef6cc641b587737e743c9a1b3", PCI device ID: , UID: "1bbe282e-b3fa-4cac-9c6d-3458fc829992", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0428 16:00:00.647389 3786 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns c5c8cbc17977a856517555f870f5be72de7ea1c4e16949b81dcf2521e3f54025 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns c5c8cbc17977a856517555f870f5be72de7ea1c4e16949b81dcf2521e3f54025 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:fc:9f:c5:39:12\",\"name\":\"c5c8cbc17977a85\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ea7aec6-2dda-4258-bef1-1364d99609e0\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 16:00:00.697493 3786 cni.go:473] [segment-bridge/segment-bridge-29623200-2fqjs ca2d92dc3fea7c503fc1a735d5c47878d1531ceef6cc641b587737e743c9a1b3 network default NAD default NAD key default] ADD finished CNI request [segment-bridge/segment-bridge-29623200-2fqjs ca2d92dc3fea7c503fc1a735d5c47878d1531ceef6cc641b587737e743c9a1b3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:31:f9:3c:6b:77\",\"name\":\"ca2d92dc3fea7c5\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5769086f-35bc-4110-a5e5-5af578d8dba6\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0428 16:00:01.027076 3786 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:01.027159 3786 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:01.029826 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-cgccv I0428 16:00:01.029853 3786 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-cgccv I0428 16:00:04.376801 3786 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns c5c8cbc17977a856517555f870f5be72de7ea1c4e16949b81dcf2521e3f54025 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns c5c8cbc17977a856517555f870f5be72de7ea1c4e16949b81dcf2521e3f54025 network default NAD default NAD key ] I0428 16:00:04.462281 3786 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns c5c8cbc17977a856517555f870f5be72de7ea1c4e16949b81dcf2521e3f54025 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns c5c8cbc17977a856517555f870f5be72de7ea1c4e16949b81dcf2521e3f54025 network default NAD default NAD key default], result "{}", err I0428 16:00:04.512648 3786 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 16:00:04.512709 3786 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns I0428 16:00:04.514191 3786 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns, ips: 10.128.0.57 I0428 16:00:04.514216 3786 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 16:00:05.539038 3786 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:05.539064 3786 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:17.102711 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0428 16:01:01.230787 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0428 16:01:03.198195 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0428 16:02:00.209221 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0428 16:02:29.195653 3786 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0428 16:03:03.113239 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=36 I0428 16:03:37.234202 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=5 I0428 16:04:32.106762 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=7 I0428 16:04:32.106761 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=9 I0428 16:04:38.197373 3786 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0428 16:04:47.112128 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=36 I0428 16:05:15.227974 3786 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0428 16:06:05.221061 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0428 16:06:07.205831 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0428 16:07:03.108317 3786 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8 I0428 16:07:13.216932 3786 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9