+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-177.ec2.internal ++ [[ -n ip-10-0-2-177.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-177.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0417 18:39:43.750424858 - disable conntrack on geneve port' I0417 18:39:43.750424858 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' + echo 'I0417 18:39:43.769798301 - starting ovnkube-node' I0417 18:39:43.769798301 - starting ovnkube-node + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-177.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-177.ec2.internal --init-node ip-10-0-2-177.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-177.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0417 18:39:43.814664 3681 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0417 18:39:43.814759 3681 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-c13051bd47.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0417 18:39:43.816399 3681 kube.go:431] Waiting for certificate I0417 18:39:43.816429 3681 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0417 18:39:43.816470 3681 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0417 18:39:43.817335 3681 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0417 18:39:43.847527 3681 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0417 18:39:43.847550 3681 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0417 18:39:43.850381 3681 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0417 18:39:43.850501 3681 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-kz5rj" I0417 18:39:43.857888 3681 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-kz5rj" I0417 18:39:43.857977 3681 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0417 18:39:44.816770 3681 kube.go:438] Certificate found I0417 18:39:44.817455 3681 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0417 18:39:44.819016 3681 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0417 18:39:44.893296 3681 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0417 18:39:44.911939 3681 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-18 18:34:43 +0000 UTC" deadline="2026-04-18 14:12:48.523817765 +0000 UTC" I0417 18:39:44.911972 3681 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h33m3.611856996s" I0417 18:39:44.916814 3681 controller_manager.go:370] Starting the ovnkube controller I0417 18:39:44.916830 3681 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-177.ec2.internal I0417 18:39:44.916900 3681 controller_manager.go:395] NBDB zone sync took: 56.308µs I0417 18:39:44.916918 3681 factory.go:561] Starting watch factory I0417 18:39:44.917082 3681 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.917154 3681 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.917081 3681 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.917359 3681 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0417 18:39:44.917378 3681 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.917401 3681 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.917354 3681 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.917483 3681 server.go:101] MetricServer registers OVS metrics I0417 18:39:44.917098 3681 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.917106 3681 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.917672 3681 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.917798 3681 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.918087 3681 server.go:105] MetricServer registers OVN DB metrics I0417 18:39:44.918886 3681 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0417 18:39:44.918967 3681 factory.go:561] Starting watch factory I0417 18:39:44.919963 3681 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.920021 3681 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.923203 3681 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0417 18:39:44.923234 3681 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0417 18:39:44.936215 3681 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.936440 3681 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.938168 3681 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0417 18:39:44.938220 3681 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0417 18:39:44.938815 3681 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.938848 3681 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.942860 3681 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.943449 3681 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0417 18:39:44.943493 3681 server.go:109] MetricServer registers OVN Controller metrics I0417 18:39:44.947610 3681 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0417 18:39:44.948836 3681 server.go:113] MetricServer registers OVN Northd metrics I0417 18:39:44.953576 3681 metrics.go:519] OVN Metrics Server starts to run ... I0417 18:39:45.022359 3681 factory.go:1977] *v1.Namespace informer cache synced successfully I0417 18:39:45.022384 3681 factory.go:1977] *v1.Node informer cache synced successfully I0417 18:39:45.022388 3681 factory.go:1977] *v1.Service informer cache synced successfully I0417 18:39:45.022391 3681 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0417 18:39:45.022394 3681 factory.go:1977] *v1.Pod informer cache synced successfully I0417 18:39:45.022397 3681 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0417 18:39:45.022513 3681 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0417 18:39:45.022540 3681 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0417 18:39:45.022513 3681 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0417 18:39:45.022602 3681 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0417 18:39:45.024594 3681 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0417 18:39:45.024605 3681 factory.go:1977] *v1.Namespace informer cache synced successfully I0417 18:39:45.024609 3681 factory.go:1977] *v1.Node informer cache synced successfully I0417 18:39:45.024612 3681 factory.go:1977] *v1.Service informer cache synced successfully I0417 18:39:45.024615 3681 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0417 18:39:45.024618 3681 factory.go:1977] *v1.Pod informer cache synced successfully I0417 18:39:45.026200 3681 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0417 18:39:45.026206 3681 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0417 18:39:45.027668 3681 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0417 18:39:45.027682 3681 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0417 18:39:45.027773 3681 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.027791 3681 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.029903 3681 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0417 18:39:45.029913 3681 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0417 18:39:45.030350 3681 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.033017 3681 factory.go:1977] *v1.EgressIP informer cache synced successfully I0417 18:39:45.033101 3681 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.033130 3681 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.035247 3681 factory.go:1977] *v1.EgressIP informer cache synced successfully I0417 18:39:45.035672 3681 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.038993 3681 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0417 18:39:45.039062 3681 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.039074 3681 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.041134 3681 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0417 18:39:45.041618 3681 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.044991 3681 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0417 18:39:45.045056 3681 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.045071 3681 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.046665 3681 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0417 18:39:45.047623 3681 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.050986 3681 factory.go:1977] *v1.EgressService informer cache synced successfully I0417 18:39:45.051064 3681 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.051101 3681 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.052125 3681 factory.go:1977] *v1.EgressService informer cache synced successfully I0417 18:39:45.053749 3681 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.057061 3681 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0417 18:39:45.057235 3681 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0417 18:39:45.057268 3681 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0417 18:39:45.057293 3681 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0417 18:39:45.059870 3681 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0417 18:39:45.062131 3681 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0417 18:39:45.062215 3681 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.062240 3681 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.062221 3681 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.062298 3681 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.062333 3681 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0417 18:39:45.065156 3681 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.065652 3681 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0417 18:39:45.067575 3681 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0417 18:39:45.067585 3681 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0417 18:39:45.067590 3681 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0417 18:39:45.067595 3681 factory.go:680] Watch Factory start up complete, took: 148.62712ms I0417 18:39:45.067600 3681 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0417 18:39:45.067607 3681 factory.go:680] Watch Factory start up complete, took: 150.693579ms I0417 18:39:45.067614 3681 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-177.ec2.internal" zone I0417 18:39:45.067676 3681 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0417 18:39:45.119410 3681 default_node_network_controller.go:710] Initializing the default node network controller I0417 18:39:45.211734 3681 udn_isolation.go:101] Starting UDN host isolation manager I0417 18:39:45.218231 3681 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0417 18:39:45.278542 3681 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0417 18:39:45.278612 3681 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0417 18:39:45.278628 3681 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0417 18:39:45.312796 3681 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0417 18:39:45.312868 3681 default_node_network_controller.go:809] Node ip-10-0-2-177.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0417 18:39:45.318598 3681 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.177"] k8s.ovn.org/zone-name:ip-10-0-2-177.ec2.internal] on node ip-10-0-2-177.ec2.internal I0417 18:39:45.339177 3681 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0417 18:39:45.345585 3681 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0417 18:39:45.345787 3681 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0417 18:39:45.345938 3681 gateway_init.go:263] Preparing Gateway I0417 18:39:45.345951 3681 gateway_shared_intf.go:1687] Creating new gateway I0417 18:39:45.457134 3681 iptables.go:108] Creating table: filter chain: FORWARD I0417 18:39:45.463703 3681 iptables.go:108] Creating table: filter chain: OUTPUT I0417 18:39:45.469982 3681 gateway_shared_intf.go:1816] Gateway Creation Complete I0417 18:39:45.470000 3681 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-177.ec2.internal","mac-address":"0e:c1:06:8e:bb:73","ip-addresses":["10.0.2.177/24"],"ip-address":"10.0.2.177/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:f46bf86f-a08d-49cb-bfdd-d33ab97e40d7 k8s.ovn.org/node-encap-ips:["10.0.2.177"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.177/24"} k8s.ovn.org/zone-name:ip-10-0-2-177.ec2.internal] on node ip-10-0-2-177.ec2.internal I0417 18:39:45.491626 3681 default_node_network_controller.go:519] Node connection status = connected I0417 18:39:45.568560 3681 controller_manager.go:429] Waiting for node in zone sync took: 500.929686ms I0417 18:39:45.574171 3681 controller_manager.go:611] Switching to transit router for layer2 networks I0417 18:39:45.574193 3681 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-177.ec2.internal I0417 18:39:45.591652 3681 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0417 18:39:45.598672 3681 controller_manager.go:290] SCTP support detected in OVN I0417 18:39:45.599306 3681 services_controller.go:69] Creating services controller for network=default I0417 18:39:45.599406 3681 udn_enabled_svc.go:80] Waiting for service informer to sync I0417 18:39:45.599496 3681 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0417 18:39:45.599528 3681 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0417 18:39:45.599542 3681 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0417 18:39:45.599552 3681 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0417 18:39:45.599739 3681 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0417 18:39:45.599783 3681 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0417 18:39:45.599818 3681 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 47.343µs I0417 18:39:45.599824 3681 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 33.617µs I0417 18:39:45.599833 3681 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 50.254µs I0417 18:39:45.599838 3681 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0417 18:39:45.599847 3681 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0417 18:39:45.599858 3681 default_network_controller.go:325] Starting the default network controller I0417 18:39:45.599870 3681 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 11.828µs I0417 18:39:45.600359 3681 udn_enabled_svc.go:95] Performing full resync I0417 18:39:45.600756 3681 udn_enabled_svc.go:99] Waiting for handler to sync I0417 18:39:45.600768 3681 udn_enabled_svc.go:104] Starting worker I0417 18:39:45.609758 3681 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0417 18:39:45.609828 3681 repair.go:33] Syncing exgw routes took 56.69µs I0417 18:39:45.609859 3681 default_network_controller.go:414] Starting all the Watchers... I0417 18:39:45.610101 3681 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0417 18:39:45.610112 3681 namespace.go:98] [openshift-node] adding namespace I0417 18:39:45.610120 3681 namespace.go:98] [openshift-dns-operator] adding namespace I0417 18:39:45.610125 3681 namespace.go:98] [openshift-authentication-operator] adding namespace I0417 18:39:45.610123 3681 namespace.go:98] [openshift-network-console] adding namespace I0417 18:39:45.610133 3681 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0417 18:39:45.610135 3681 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0417 18:39:45.610145 3681 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0417 18:39:45.610147 3681 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0417 18:39:45.610148 3681 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0417 18:39:45.610155 3681 namespace.go:98] [openshift-customer-monitoring] adding namespace I0417 18:39:45.610151 3681 namespace.go:98] [default] adding namespace I0417 18:39:45.610168 3681 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0417 18:39:45.610174 3681 namespace.go:98] [openshift-package-operator] adding namespace I0417 18:39:45.610105 3681 namespace.go:98] [openshift-console-user-settings] adding namespace I0417 18:39:45.610717 3681 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 603.503µs I0417 18:39:45.610732 3681 namespace.go:98] [open-cluster-management-2po2q8b3ubd89mbta5n7l49aiaaqqcse] adding namespace I0417 18:39:45.611193 3681 namespace.go:102] [open-cluster-management-2po2q8b3ubd89mbta5n7l49aiaaqqcse] adding namespace took 455.571µs I0417 18:39:45.611206 3681 namespace.go:98] [openshift-ingress-canary] adding namespace I0417 18:39:45.611715 3681 namespace.go:102] [openshift-ingress-canary] adding namespace took 504.042µs I0417 18:39:45.611727 3681 namespace.go:98] [openshift-kube-apiserver] adding namespace I0417 18:39:45.612193 3681 namespace.go:102] [openshift-node] adding namespace took 2.070469ms I0417 18:39:45.612210 3681 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0417 18:39:45.612624 3681 namespace.go:102] [openshift-dns-operator] adding namespace took 2.494757ms I0417 18:39:45.612639 3681 namespace.go:98] [openshift-marketplace] adding namespace I0417 18:39:45.613079 3681 namespace.go:102] [openshift-authentication-operator] adding namespace took 2.948735ms I0417 18:39:45.613090 3681 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0417 18:39:45.613508 3681 namespace.go:102] [openshift-network-console] adding namespace took 3.373852ms I0417 18:39:45.613522 3681 namespace.go:98] [openshift] adding namespace I0417 18:39:45.613986 3681 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 3.849054ms I0417 18:39:45.613996 3681 namespace.go:98] [openshift-operators] adding namespace I0417 18:39:45.614470 3681 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 4.326852ms I0417 18:39:45.614484 3681 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0417 18:39:45.614983 3681 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 4.831859ms I0417 18:39:45.614994 3681 namespace.go:98] [openshift-config-operator] adding namespace I0417 18:39:45.615479 3681 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 5.326175ms I0417 18:39:45.615488 3681 namespace.go:98] [openshift-backplane] adding namespace I0417 18:39:45.615984 3681 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 5.824321ms I0417 18:39:45.616000 3681 namespace.go:98] [openshift-network-operator] adding namespace I0417 18:39:45.616452 3681 namespace.go:102] [openshift-customer-monitoring] adding namespace took 6.291431ms I0417 18:39:45.616463 3681 namespace.go:98] [openshift-service-ca-operator] adding namespace I0417 18:39:45.616888 3681 namespace.go:102] [default] adding namespace took 6.724206ms I0417 18:39:45.616898 3681 namespace.go:98] [openshift-backplane-cse] adding namespace I0417 18:39:45.617446 3681 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.267964ms I0417 18:39:45.617456 3681 namespace.go:98] [openshift-insights] adding namespace I0417 18:39:45.617939 3681 namespace.go:102] [openshift-package-operator] adding namespace took 7.753911ms I0417 18:39:45.617957 3681 namespace.go:98] [openshift-route-controller-manager] adding namespace I0417 18:39:45.618369 3681 namespace.go:102] [openshift-console-user-settings] adding namespace took 8.143024ms I0417 18:39:45.618381 3681 namespace.go:98] [openshift-backplane-csm] adding namespace I0417 18:39:45.618841 3681 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.107463ms I0417 18:39:45.618855 3681 namespace.go:98] [openshift-kube-scheduler] adding namespace I0417 18:39:45.619204 3681 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.988078ms I0417 18:39:45.619221 3681 namespace.go:98] [openshift-cluster-version] adding namespace I0417 18:39:45.619671 3681 namespace.go:102] [openshift-marketplace] adding namespace took 7.025737ms I0417 18:39:45.619686 3681 namespace.go:98] [openshift-infra] adding namespace I0417 18:39:45.620209 3681 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 7.113456ms I0417 18:39:45.620222 3681 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0417 18:39:45.620595 3681 namespace.go:102] [openshift] adding namespace took 7.067648ms I0417 18:39:45.620605 3681 namespace.go:98] [openshift-console] adding namespace I0417 18:39:45.621097 3681 namespace.go:102] [openshift-operators] adding namespace took 7.092087ms I0417 18:39:45.621115 3681 namespace.go:98] [openshift-console-operator] adding namespace I0417 18:39:45.621574 3681 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 7.084952ms I0417 18:39:45.621586 3681 namespace.go:98] [openshift-dns] adding namespace I0417 18:39:45.621990 3681 namespace.go:102] [openshift-config-operator] adding namespace took 6.99067ms I0417 18:39:45.622003 3681 namespace.go:98] [openshift-monitoring] adding namespace I0417 18:39:45.622466 3681 namespace.go:102] [openshift-backplane] adding namespace took 6.972861ms I0417 18:39:45.622479 3681 namespace.go:98] [openshift-network-node-identity] adding namespace I0417 18:39:45.622874 3681 namespace.go:102] [openshift-network-operator] adding namespace took 6.869947ms I0417 18:39:45.622886 3681 namespace.go:98] [openshift-backplane-mobb] adding namespace I0417 18:39:45.623345 3681 namespace.go:102] [openshift-service-ca-operator] adding namespace took 6.876731ms I0417 18:39:45.623357 3681 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0417 18:39:45.623729 3681 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.82672ms I0417 18:39:45.623739 3681 namespace.go:98] [openshift-apiserver-operator] adding namespace I0417 18:39:45.624285 3681 namespace.go:102] [openshift-insights] adding namespace took 6.825615ms I0417 18:39:45.624294 3681 namespace.go:98] [openshift-backplane-cee] adding namespace I0417 18:39:45.624685 3681 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.722121ms I0417 18:39:45.624715 3681 namespace.go:98] [openshift-ingress-operator] adding namespace I0417 18:39:45.625193 3681 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.806239ms I0417 18:39:45.625206 3681 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0417 18:39:45.626149 3681 namespace.go:102] [openshift-kube-scheduler] adding namespace took 7.28568ms I0417 18:39:45.626168 3681 namespace.go:98] [kube-system] adding namespace I0417 18:39:45.626596 3681 namespace.go:102] [openshift-cluster-version] adding namespace took 7.369861ms I0417 18:39:45.626609 3681 namespace.go:98] [kube-node-lease] adding namespace I0417 18:39:45.626991 3681 namespace.go:102] [openshift-infra] adding namespace took 7.284635ms I0417 18:39:45.627003 3681 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0417 18:39:45.627483 3681 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.253799ms I0417 18:39:45.627499 3681 namespace.go:98] [openshift-multus] adding namespace I0417 18:39:45.627944 3681 namespace.go:102] [openshift-console] adding namespace took 7.333693ms I0417 18:39:45.627961 3681 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0417 18:39:45.628330 3681 namespace.go:102] [openshift-console-operator] adding namespace took 7.209183ms I0417 18:39:45.628345 3681 namespace.go:98] [openshift-logging] adding namespace I0417 18:39:45.628842 3681 namespace.go:102] [openshift-dns] adding namespace took 7.249597ms I0417 18:39:45.628861 3681 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0417 18:39:45.629253 3681 namespace.go:102] [openshift-monitoring] adding namespace took 7.244256ms I0417 18:39:45.629268 3681 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0417 18:39:45.629678 3681 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.192627ms I0417 18:39:45.629711 3681 namespace.go:98] [openshift-apiserver] adding namespace I0417 18:39:45.630112 3681 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.22075ms I0417 18:39:45.630127 3681 namespace.go:98] [openshift-controller-manager] adding namespace I0417 18:39:45.630567 3681 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 7.201751ms I0417 18:39:45.630584 3681 namespace.go:98] [openshift-image-registry] adding namespace I0417 18:39:45.630985 3681 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.240573ms I0417 18:39:45.631000 3681 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0417 18:39:45.631377 3681 namespace.go:102] [openshift-backplane-cee] adding namespace took 7.077046ms I0417 18:39:45.631392 3681 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0417 18:39:45.631794 3681 namespace.go:102] [openshift-ingress-operator] adding namespace took 7.072731ms I0417 18:39:45.631808 3681 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0417 18:39:45.632217 3681 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 7.00503ms I0417 18:39:45.632233 3681 namespace.go:98] [openshift-machine-api] adding namespace I0417 18:39:45.632721 3681 namespace.go:102] [kube-system] adding namespace took 6.545247ms I0417 18:39:45.633142 3681 namespace.go:102] [kube-node-lease] adding namespace took 6.526722ms I0417 18:39:45.633156 3681 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace W0417 18:39:45.633565 3681 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-lp6tg: pod openshift-multus/network-metrics-daemon-lp6tg: no pod IPs found I0417 18:39:45.633610 3681 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.601319ms I0417 18:39:45.633625 3681 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0417 18:39:45.634045 3681 namespace.go:102] [openshift-multus] adding namespace took 6.537891ms I0417 18:39:45.634065 3681 namespace.go:98] [kube-public] adding namespace I0417 18:39:45.634519 3681 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.55045ms I0417 18:39:45.634538 3681 namespace.go:98] [openshift-ingress] adding namespace I0417 18:39:45.634967 3681 namespace.go:102] [openshift-logging] adding namespace took 6.615572ms I0417 18:39:45.634983 3681 namespace.go:98] [openshift-must-gather-operator] adding namespace I0417 18:39:45.635421 3681 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.553336ms I0417 18:39:45.635436 3681 namespace.go:98] [openshift-config] adding namespace I0417 18:39:45.635875 3681 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.599842ms I0417 18:39:45.635890 3681 namespace.go:98] [openshift-machine-config-operator] adding namespace I0417 18:39:45.636323 3681 namespace.go:102] [openshift-apiserver] adding namespace took 6.605115ms I0417 18:39:45.636338 3681 namespace.go:98] [openshift-etcd] adding namespace I0417 18:39:45.636776 3681 namespace.go:102] [openshift-controller-manager] adding namespace took 6.641778ms I0417 18:39:45.636791 3681 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0417 18:39:45.637231 3681 namespace.go:102] [openshift-image-registry] adding namespace took 6.641919ms I0417 18:39:45.637245 3681 namespace.go:98] [openshift-authentication] adding namespace I0417 18:39:45.637653 3681 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.647075ms I0417 18:39:45.637666 3681 namespace.go:98] [openshift-host-network] adding namespace I0417 18:39:45.638110 3681 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.711762ms I0417 18:39:45.638123 3681 namespace.go:98] [openshift-operators-redhat] adding namespace I0417 18:39:45.638509 3681 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.696069ms I0417 18:39:45.638522 3681 namespace.go:98] [openshift-backplane-srep] adding namespace I0417 18:39:45.638960 3681 namespace.go:102] [openshift-machine-api] adding namespace took 6.721045ms I0417 18:39:45.638976 3681 namespace.go:98] [openshift-network-diagnostics] adding namespace I0417 18:39:45.639329 3681 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.167178ms I0417 18:39:45.639343 3681 namespace.go:98] [dedicated-admin] adding namespace I0417 18:39:45.639810 3681 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.179403ms I0417 18:39:45.640160 3681 namespace.go:102] [kube-public] adding namespace took 6.090049ms I0417 18:39:45.640607 3681 namespace.go:102] [openshift-ingress] adding namespace took 6.058861ms I0417 18:39:45.640620 3681 namespace.go:98] [openshift-backplane-tam] adding namespace I0417 18:39:45.641016 3681 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.027681ms I0417 18:39:45.641421 3681 namespace.go:102] [openshift-config] adding namespace took 5.979439ms I0417 18:39:45.641906 3681 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.010614ms I0417 18:39:45.642306 3681 namespace.go:102] [openshift-etcd] adding namespace took 5.961921ms I0417 18:39:45.642713 3681 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 5.916555ms I0417 18:39:45.643181 3681 namespace.go:102] [openshift-authentication] adding namespace took 5.931804ms I0417 18:39:45.643651 3681 namespace.go:102] [openshift-host-network] adding namespace took 5.979511ms I0417 18:39:45.644160 3681 namespace.go:102] [openshift-operators-redhat] adding namespace took 6.032051ms W0417 18:39:45.644565 3681 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-lkw5s: pod openshift-network-diagnostics/network-check-target-lkw5s: no pod IPs found I0417 18:39:45.644611 3681 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.082188ms I0417 18:39:45.645185 3681 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.198942ms I0417 18:39:45.645203 3681 namespace.go:98] [openshift-config-managed] adding namespace I0417 18:39:45.645682 3681 namespace.go:102] [dedicated-admin] adding namespace took 6.333959ms I0417 18:39:45.646108 3681 namespace.go:102] [openshift-backplane-tam] adding namespace took 5.482687ms I0417 18:39:45.646462 3681 namespace.go:102] [openshift-config-managed] adding namespace took 1.253044ms I0417 18:39:45.646643 3681 master.go:548] Adding or Updating local node "ip-10-0-2-177.ec2.internal" for network "default" W0417 18:39:45.654305 3681 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-lp6tg: pod openshift-multus/network-metrics-daemon-lp6tg: no pod IPs found W0417 18:39:45.654325 3681 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-lkw5s: pod openshift-network-diagnostics/network-check-target-lkw5s: no pod IPs found I0417 18:39:45.655337 3681 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-177.ec2.internal for the network default I0417 18:39:45.655606 3681 zone_ic_handler.go:195] Time taken to create transit switch: 257.555µs I0417 18:39:45.656335 3681 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0417 18:39:45.656359 3681 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0417 18:39:45.656374 3681 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0417 18:39:45.656384 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-177.ec2.internal in network "default" I0417 18:39:45.656439 3681 node_tracker.go:169] Node ip-10-0-2-177.ec2.internal switch + router changed, syncing services in network "default" I0417 18:39:45.656455 3681 services_controller.go:574] Full service sync requested for network=default I0417 18:39:45.757439 3681 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0417 18:39:45.757466 3681 services_controller.go:208] Setting up event handlers for services for network=default I0417 18:39:45.757534 3681 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0417 18:39:45.757591 3681 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0417 18:39:45.757609 3681 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0417 18:39:45.858465 3681 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0417 18:39:45.858494 3681 repair.go:60] Starting repairing loop for services I0417 18:39:45.858598 3681 repair.go:131] Deleted 0 stale service LBs I0417 18:39:45.858617 3681 repair.go:137] Deleted 0 stale Chassis Template Vars I0417 18:39:45.858635 3681 repair.go:62] Finished repairing loop for services: 145.882µs I0417 18:39:45.858659 3681 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0417 18:39:45.858669 3681 services_controller.go:254] Starting workers for network=default I0417 18:39:45.859015 3681 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-lp6tg] creating logical port openshift-multus_network-metrics-daemon-lp6tg for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:45.859022 3681 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-lkw5s] creating logical port openshift-network-diagnostics_network-check-target-lkw5s for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:45.859183 3681 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-lp6tg I0417 18:39:45.859232 3681 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-lkw5s I0417 18:39:45.876991 3681 pod.go:62] [openshift-network-diagnostics/network-check-target-lkw5s] pod update took 17.787706ms I0417 18:39:45.877019 3681 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-lkw5s] addLogicalPort annotation time took 17.817951ms I0417 18:39:45.877126 3681 pod.go:62] [openshift-multus/network-metrics-daemon-lp6tg] pod update took 18.032054ms I0417 18:39:45.877144 3681 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-lp6tg] addLogicalPort annotation time took 18.054509ms I0417 18:39:45.878004 3681 pods.go:271] [openshift-network-diagnostics/network-check-target-lkw5s] addLogicalPort took 18.989386ms, libovsdb time 712.344µs I0417 18:39:45.878309 3681 pods.go:271] [openshift-multus/network-metrics-daemon-lp6tg] addLogicalPort took 19.308344ms, libovsdb time 892.905µs I0417 18:39:45.879194 3681 admin_network_policy_controller.go:227] Starting controller default-network-controller I0417 18:39:45.879218 3681 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0417 18:39:45.879235 3681 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0417 18:39:45.879243 3681 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0417 18:39:45.879432 3681 repair.go:29] Repairing admin network policies took 183.095µs I0417 18:39:45.879541 3681 repair.go:92] Repairing baseline admin network policies took 99.358µs I0417 18:39:45.879948 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0417 18:39:45.879965 3681 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0417 18:39:45.881537 3681 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0417 18:39:45.881564 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0417 18:39:45.881578 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0417 18:39:45.883087 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0417 18:39:45.883112 3681 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0417 18:39:45.883123 3681 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0417 18:39:45.884350 3681 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0417 18:39:45.884373 3681 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0417 18:39:45.884387 3681 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0417 18:39:45.885280 3681 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0417 18:39:45.885305 3681 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0417 18:39:45.885314 3681 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0417 18:39:45.886565 3681 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0417 18:39:45.886584 3681 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0417 18:39:45.886593 3681 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0417 18:39:45.888234 3681 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0417 18:39:45.888257 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0417 18:39:45.888267 3681 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0417 18:39:45.888538 3681 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0417 18:39:45.888558 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0417 18:39:45.888567 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0417 18:39:45.889050 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0417 18:39:45.889064 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0417 18:39:45.889074 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0417 18:39:45.890562 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0417 18:39:45.890586 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0417 18:39:45.890594 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0417 18:39:45.890892 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0417 18:39:45.890917 3681 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0417 18:39:45.890930 3681 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0417 18:39:45.893141 3681 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0417 18:39:45.893162 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0417 18:39:45.893174 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0417 18:39:45.893878 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0417 18:39:45.893898 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0417 18:39:45.893907 3681 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0417 18:39:45.894413 3681 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0417 18:39:45.894436 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0417 18:39:45.895037 3681 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0417 18:39:45.895064 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0417 18:39:45.895861 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0417 18:39:45.895881 3681 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0417 18:39:45.895895 3681 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0417 18:39:45.897144 3681 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0417 18:39:45.897167 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0417 18:39:45.897180 3681 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0417 18:39:45.897949 3681 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0417 18:39:45.897967 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0417 18:39:45.897975 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0417 18:39:45.898566 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0417 18:39:45.898583 3681 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0417 18:39:45.898592 3681 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0417 18:39:45.899940 3681 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0417 18:39:45.899962 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0417 18:39:45.899970 3681 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0417 18:39:45.900562 3681 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0417 18:39:45.900580 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0417 18:39:45.900602 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0417 18:39:45.901402 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0417 18:39:45.901421 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0417 18:39:45.901443 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0417 18:39:45.901971 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0417 18:39:45.901989 3681 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0417 18:39:45.901998 3681 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0417 18:39:45.903103 3681 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0417 18:39:45.903121 3681 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0417 18:39:45.903130 3681 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0417 18:39:45.904860 3681 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0417 18:39:45.904878 3681 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0417 18:39:45.904885 3681 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0417 18:39:45.905355 3681 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0417 18:39:45.905372 3681 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0417 18:39:45.905382 3681 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0417 18:39:45.906343 3681 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0417 18:39:45.906361 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0417 18:39:45.906369 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0417 18:39:45.906856 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0417 18:39:45.906874 3681 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0417 18:39:45.906880 3681 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0417 18:39:45.908179 3681 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0417 18:39:45.908204 3681 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0417 18:39:45.908216 3681 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0417 18:39:45.908844 3681 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0417 18:39:45.908861 3681 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0417 18:39:45.908873 3681 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0417 18:39:45.909833 3681 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0417 18:39:45.914484 3681 egressfirewall.go:374] Starting EgressFirewall controller I0417 18:39:45.914525 3681 controller.go:133] Adding controller egress-firewall-controller event handlers I0417 18:39:45.914557 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0417 18:39:45.914571 3681 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0417 18:39:45.914527 3681 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0417 18:39:45.914590 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0417 18:39:45.914598 3681 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0417 18:39:45.914898 3681 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0417 18:39:45.914912 3681 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0417 18:39:45.914918 3681 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0417 18:39:45.914927 3681 egressqos.go:193] Setting up event handlers for EgressQoS I0417 18:39:45.914938 3681 egressfirewall.go:723] Syncing node "ip-10-0-2-177.ec2.internal" for egress firewall I0417 18:39:45.915010 3681 egressqos.go:245] Starting EgressQoS Controller I0417 18:39:45.915020 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0417 18:39:45.915028 3681 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0417 18:39:45.915034 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0417 18:39:45.915039 3681 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0417 18:39:45.915044 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0417 18:39:45.915048 3681 shared_informer.go:356] "Caches are synced" controller="egressqos" I0417 18:39:45.915052 3681 egressqos.go:259] Repairing EgressQoSes I0417 18:39:45.915056 3681 egressqos.go:399] Starting repairing loop for egressqos I0417 18:39:45.915123 3681 egressqos.go:401] Finished repairing loop for egressqos: 67.025µs I0417 18:39:45.915143 3681 egressservice_zone.go:132] Setting up event handlers for Egress Services I0417 18:39:45.915169 3681 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-177.ec2.internal I0417 18:39:45.915239 3681 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-177.ec2.internal : 71.592µs I0417 18:39:45.915412 3681 egressservice_zone.go:210] Starting Egress Services Controller I0417 18:39:45.915428 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0417 18:39:45.915443 3681 shared_informer.go:356] "Caches are synced" controller="egressservices" I0417 18:39:45.915452 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0417 18:39:45.915460 3681 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0417 18:39:45.915467 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0417 18:39:45.915472 3681 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0417 18:39:45.915478 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0417 18:39:45.915482 3681 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0417 18:39:45.915486 3681 egressservice_zone.go:228] Repairing Egress Services I0417 18:39:45.915892 3681 master_controller.go:88] Starting Admin Policy Based Route Controller I0417 18:39:45.915902 3681 external_controller.go:278] Starting Admin Policy Based Route Controller I0417 18:39:45.915971 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-177.ec2.internal I0417 18:39:45.915988 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-177.ec2.internal: 20.049µs I0417 18:39:45.916087 3681 default_network_controller.go:585] Completing all the Watchers took 306.215287ms I0417 18:39:45.916110 3681 default_network_controller.go:589] Starting unidling controllers I0417 18:39:45.916132 3681 unidle.go:46] Registering OVN SB ControllerEvent handler I0417 18:39:45.916141 3681 unidle.go:63] Populating Initial ContollerEvent events I0417 18:39:45.916152 3681 unidle.go:79] Setting up event handlers for services I0417 18:39:45.916243 3681 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0417 18:39:45.992365 3681 default_node_network_controller.go:519] Node connection status = connected I0417 18:39:46.492471 3681 default_node_network_controller.go:519] Node connection status = connected I0417 18:39:46.992841 3681 default_node_network_controller.go:519] Node connection status = connected I0417 18:39:47.492402 3681 default_node_network_controller.go:519] Node connection status = connected I0417 18:39:47.761814 3681 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0417 18:39:47.991781 3681 default_node_network_controller.go:519] Node connection status = connected I0417 18:39:48.010290 3681 bridgeconfig.go:537] Gateway is ready I0417 18:39:48.010313 3681 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0417 18:39:48.022619 3681 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.177. Updating OVN encap IP. I0417 18:39:48.028461 3681 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.177 - it is already configured I0417 18:39:48.028478 3681 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.177/24:{} 172.20.0.1/32:{}]. Updating annotations. I0417 18:39:48.028760 3681 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.177/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-177.ec2.internal","mac-address":"0e:c1:06:8e:bb:73","ip-addresses":["10.0.2.177/24"],"ip-address":"10.0.2.177/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:f46bf86f-a08d-49cb-bfdd-d33ab97e40d7 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.177/24"}] on node ip-10-0-2-177.ec2.internal I0417 18:39:48.045581 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-177.ec2.internal in network "default" I0417 18:39:48.045612 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-177.ec2.internal I0417 18:39:48.045626 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-177.ec2.internal: 19.424µs I0417 18:39:48.045634 3681 egressfirewall.go:723] Syncing node "ip-10-0-2-177.ec2.internal" for egress firewall I0417 18:39:48.045646 3681 node_tracker.go:169] Node ip-10-0-2-177.ec2.internal switch + router changed, syncing services in network "default" I0417 18:39:48.045660 3681 services_controller.go:574] Full service sync requested for network=default I0417 18:39:48.045665 3681 master.go:548] Adding or Updating local node "ip-10-0-2-177.ec2.internal" for network "default" I0417 18:39:48.046303 3681 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.177 I0417 18:39:48.046389 3681 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-177.ec2.internal I0417 18:39:48.050172 3681 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-177.ec2.internal. Updating no re-route policies I0417 18:39:48.050252 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-p5mwh I0417 18:39:48.050269 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-p5mwh I0417 18:39:48.050297 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-p5mwh after 0 failed attempt(s) I0417 18:39:48.050308 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-t85dj I0417 18:39:48.050315 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-z7b5r I0417 18:39:48.050318 3681 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-177.ec2.internal I0417 18:39:48.050324 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-z7b5r I0417 18:39:48.050330 3681 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-177.ec2.internal I0417 18:39:48.050335 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-gxcxf I0417 18:39:48.050337 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-lp6tg I0417 18:39:48.050342 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-z7b5r after 0 failed attempt(s) I0417 18:39:48.050345 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-gxcxf I0417 18:39:48.050324 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-t85dj I0417 18:39:48.050355 3681 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-177.ec2.internal after 0 failed attempt(s) I0417 18:39:48.050355 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ltlwp I0417 18:39:48.050357 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-177.ec2.internal I0417 18:39:48.050364 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-gxcxf after 0 failed attempt(s) I0417 18:39:48.050369 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-177.ec2.internal I0417 18:39:48.050370 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ltlwp I0417 18:39:48.050371 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-2td75 I0417 18:39:48.050370 3681 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-92gkz I0417 18:39:48.050378 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-8r92h I0417 18:39:48.050381 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-2td75 I0417 18:39:48.050383 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-pgnkh I0417 18:39:48.050385 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-177.ec2.internal after 0 failed attempt(s) I0417 18:39:48.050387 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-ltlwp after 0 failed attempt(s) I0417 18:39:48.050389 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-8r92h I0417 18:39:48.050393 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-pgnkh I0417 18:39:48.050401 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-2td75 after 0 failed attempt(s) I0417 18:39:48.050365 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-t85dj after 0 failed attempt(s) I0417 18:39:48.050410 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-pgnkh after 0 failed attempt(s) I0417 18:39:48.050402 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-8r92h after 0 failed attempt(s) I0417 18:39:48.050385 3681 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-92gkz I0417 18:39:48.050430 3681 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-92gkz after 0 failed attempt(s) I0417 18:39:48.050349 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-lp6tg I0417 18:39:48.050461 3681 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-lp6tg] creating logical port openshift-multus_network-metrics-daemon-lp6tg for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:48.050371 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-lkw5s I0417 18:39:48.050531 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-lkw5s I0417 18:39:48.050575 3681 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-lkw5s] creating logical port openshift-network-diagnostics_network-check-target-lkw5s for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:48.050829 3681 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/ecr-credential-refresh-initial-4g8fb I0417 18:39:48.050849 3681 obj_retry.go:398] Adding new object: *v1.Pod kube-system/ecr-credential-refresh-initial-4g8fb I0417 18:39:48.050865 3681 obj_retry.go:418] Retry successful for *v1.Pod kube-system/ecr-credential-refresh-initial-4g8fb after 0 failed attempt(s) I0417 18:39:48.051125 3681 pods.go:271] [openshift-multus/network-metrics-daemon-lp6tg] addLogicalPort took 674.206µs, libovsdb time 274.858µs I0417 18:39:48.051134 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-lp6tg after 0 failed attempt(s) I0417 18:39:48.051454 3681 pods.go:271] [openshift-network-diagnostics/network-check-target-lkw5s] addLogicalPort took 887.764µs, libovsdb time 457.059µs I0417 18:39:48.051470 3681 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-lkw5s after 0 failed attempt(s) I0417 18:39:48.062462 3681 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0417 18:39:48.074732 3681 iptables.go:108] Creating table: mangle chain: OUTPUT I0417 18:39:48.078119 3681 iptables.go:108] Creating table: nat chain: OUTPUT I0417 18:39:48.126141 3681 iptables.go:108] Creating table: nat chain: PREROUTING I0417 18:39:48.189815 3681 iptables.go:108] Creating table: filter chain: FORWARD I0417 18:39:48.206435 3681 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0417 18:39:48.206524 3681 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0417 18:39:48.206570 3681 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0417 18:39:48.206636 3681 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0417 18:39:48.206710 3681 gateway_init.go:325] Gateway and management port readiness took 2.719353427s I0417 18:39:48.206740 3681 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0417 18:39:48.206777 3681 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0417 18:39:48.206791 3681 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0417 18:39:48.206800 3681 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0417 18:39:48.206857 3681 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0417 18:39:48.206947 3681 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 60.417µs I0417 18:39:48.207141 3681 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0417 18:39:48.207167 3681 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.83µs I0417 18:39:48.207176 3681 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 34.219µs I0417 18:39:48.207182 3681 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0417 18:39:48.207191 3681 nad_controller.go:379] [node-nad-controller NAD controller]: started I0417 18:39:48.207196 3681 default_node_network_controller.go:899] Starting the default node network controller I0417 18:39:48.207201 3681 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0417 18:39:48.207277 3681 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 13.738µs I0417 18:39:48.207334 3681 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0417 18:39:48.207342 3681 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0417 18:39:48.207367 3681 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bc2940 0xc000342310 0xc000342380 0xc0003423f0 0xc000342460 0xc0003425b0 0xc000342620 0xc000c205a0 0xc000342690 0xc000342700 0xc000aaa2d0 0xc00655d340 0xc000342770 map[0x2ff9080:0xc002f16540 0x2ff95c0:0xc002f166c0 0x2ffa2e0:0xc002f16480 0x2ffa580:0xc002f164e0 0x2ffad60:0xc002f165a0 0x2ffb000:0xc002f16600 0x2ffb2a0:0xc002f16660 0x2ffba80:0xc002f16c60 0x2ffbd20:0xc002f16cc0 0x3026980:0xc000e0c840 0x3026d20:0xc000e0c960 0x3027800:0xc000c26480 0x3029160:0xc001ca2000 0x30298a0:0xc000e0d440 0x302a720:0xc000e0c7e0] 0xc0005faaf0 0} I0417 18:39:48.207428 3681 gateway.go:151] Starting gateway service sync I0417 18:39:48.207849 3681 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0417 18:39:48.210043 3681 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0417 18:39:48.237541 3681 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0417 18:39:48.238794 3681 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0417 18:39:48.240019 3681 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0417 18:39:48.275155 3681 gateway.go:168] Gateway service sync done. Time taken: 67.713155ms I0417 18:39:48.275543 3681 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 31669 I0417 18:39:48.275967 3681 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bc2940 0xc000342310 0xc000342380 0xc0003423f0 0xc000342460 0xc0003425b0 0xc000342620 0xc000c205a0 0xc000342690 0xc000342700 0xc000aaa2d0 0xc00655d340 0xc000342770 map[0x2ff9080:0xc002f16540 0x2ff95c0:0xc002f166c0 0x2ffa2e0:0xc002f16480 0x2ffa580:0xc002f164e0 0x2ffad60:0xc002f165a0 0x2ffb000:0xc002f16600 0x2ffb2a0:0xc002f16660 0x2ffba80:0xc002f16c60 0x2ffbd20:0xc002f16cc0 0x3026980:0xc000e0c840 0x3026d20:0xc000e0c960 0x3027800:0xc000c26480 0x3029160:0xc001ca2000 0x30298a0:0xc000e0d440 0x302a720:0xc000e0c7e0] 0xc0005faaf0 0} I0417 18:39:48.276394 3681 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000bc2940 0xc000342310 0xc000342380 0xc0003423f0 0xc000342460 0xc0003425b0 0xc000342620 0xc000c205a0 0xc000342690 0xc000342700 0xc000aaa2d0 0xc00655d340 0xc000342770 map[0x2ff9080:0xc002f16540 0x2ff95c0:0xc002f166c0 0x2ffa2e0:0xc002f16480 0x2ffa580:0xc002f164e0 0x2ffad60:0xc002f165a0 0x2ffb000:0xc002f16600 0x2ffb2a0:0xc002f16660 0x2ffba80:0xc002f16c60 0x2ffbd20:0xc002f16cc0 0x3026980:0xc000e0c840 0x3026d20:0xc000e0c960 0x3027800:0xc000c26480 0x3029160:0xc001ca2000 0x30298a0:0xc000e0d440 0x302a720:0xc000e0c7e0] 0xc0005faaf0 0} I0417 18:39:48.276429 3681 gateway.go:532] Reconciling gateway with updates I0417 18:39:48.282161 3681 gateway.go:339] Spawning Conntrack Rule Check Thread I0417 18:39:48.282242 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0417 18:39:48.282257 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0417 18:39:48.282266 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0417 18:39:48.282265 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0417 18:39:48.282272 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0417 18:39:48.282278 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0417 18:39:48.282284 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0417 18:39:48.282292 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0417 18:39:48.282294 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0417 18:39:48.282299 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0417 18:39:48.282304 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0417 18:39:48.282305 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0417 18:39:48.282303 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0417 18:39:48.282319 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0417 18:39:48.282321 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0417 18:39:48.282323 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0417 18:39:48.282319 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0417 18:39:48.282326 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0417 18:39:48.282332 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0417 18:39:48.282335 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0417 18:39:48.282339 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0417 18:39:48.282345 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0417 18:39:48.282349 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0417 18:39:48.282265 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0417 18:39:48.282353 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0417 18:39:48.282359 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0417 18:39:48.282361 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0417 18:39:48.282245 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0417 18:39:48.282368 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0417 18:39:48.282373 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0417 18:39:48.282376 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0417 18:39:48.282282 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0417 18:39:48.282252 3681 default_node_network_controller.go:944] Gateway and management port readiness took 74.903308ms I0417 18:39:48.282383 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0417 18:39:48.282384 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0417 18:39:48.282394 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0417 18:39:48.282396 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0417 18:39:48.282400 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0417 18:39:48.282403 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0417 18:39:48.282410 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0417 18:39:48.282406 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0417 18:39:48.282414 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0417 18:39:48.282407 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0417 18:39:48.282423 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0417 18:39:48.282360 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0417 18:39:48.282427 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0417 18:39:48.282426 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0417 18:39:48.282433 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0417 18:39:48.282437 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0417 18:39:48.282433 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0417 18:39:48.282443 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0417 18:39:48.282349 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0417 18:39:48.282453 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0417 18:39:48.282456 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0417 18:39:48.282465 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0417 18:39:48.282467 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0417 18:39:48.282468 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0417 18:39:48.282469 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0417 18:39:48.282470 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0417 18:39:48.282479 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0417 18:39:48.282481 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0417 18:39:48.282485 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0417 18:39:48.282486 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0417 18:39:48.282487 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0417 18:39:48.282403 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0417 18:39:48.282497 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0417 18:39:48.282499 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0417 18:39:48.282358 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0417 18:39:48.282509 3681 port_claim.go:71] Svc openshift-ingress/router-default: port 31836 is already open, no action needed I0417 18:39:48.282518 3681 port_claim.go:71] Svc openshift-ingress/router-default: port 32131 is already open, no action needed I0417 18:39:48.282525 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0417 18:39:48.282357 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0417 18:39:48.282395 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0417 18:39:48.282543 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0417 18:39:48.282547 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0417 18:39:48.282379 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0417 18:39:48.282559 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0417 18:39:48.282466 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0417 18:39:48.282571 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0417 18:39:48.282544 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0417 18:39:48.282408 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0417 18:39:48.282611 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0417 18:39:48.282365 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0417 18:39:48.282421 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0417 18:39:48.282371 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0417 18:39:48.282372 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0417 18:39:48.282663 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0417 18:39:48.282381 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0417 18:39:48.282384 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0417 18:39:48.282756 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0417 18:39:48.282386 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0417 18:39:48.282803 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0417 18:39:48.282388 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0417 18:39:48.282395 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0417 18:39:48.282416 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0417 18:39:48.282850 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0417 18:39:48.282435 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0417 18:39:48.282883 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0417 18:39:48.282440 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0417 18:39:48.282886 3681 node_ip_handler_linux.go:160] Node IP manager is running I0417 18:39:48.282446 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0417 18:39:48.282451 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0417 18:39:48.282454 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0417 18:39:48.282943 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0417 18:39:48.282454 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0417 18:39:48.282454 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0417 18:39:48.282983 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0417 18:39:48.282465 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0417 18:39:48.283013 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0417 18:39:48.282471 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0417 18:39:48.283106 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0417 18:39:48.282475 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0417 18:39:48.282473 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0417 18:39:48.282480 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0417 18:39:48.282370 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0417 18:39:48.283236 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0417 18:39:48.282894 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0417 18:39:48.282439 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0417 18:39:48.283548 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics W0417 18:39:48.321294 3681 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0417 18:39:48.321407 3681 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0417 18:39:48.321532 3681 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0417 18:39:48.329419 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0417 18:39:48.329435 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0417 18:39:48.329454 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0417 18:39:48.329461 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0417 18:39:48.329467 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0417 18:39:48.329475 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0417 18:39:48.370454 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0417 18:39:48.370481 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0417 18:39:48.370489 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0417 18:39:48.410343 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0417 18:39:48.410373 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0417 18:39:48.410381 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0417 18:39:48.410388 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0417 18:39:48.410395 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0417 18:39:48.410400 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0417 18:39:48.410416 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0417 18:39:48.410434 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0417 18:39:49.537309 3681 default_node_network_controller.go:1380] Starting node controller node sync W0417 18:39:49.537320 3681 egressip_healthcheck.go:76] Health checking using insecure connection I0417 18:39:49.537384 3681 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0417 18:39:49.549188 3681 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.853341ms I0417 18:39:49.549335 3681 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0417 18:39:49.549673 3681 egressservice_node.go:93] Setting up event handlers for Egress Services I0417 18:39:49.549847 3681 egressservice_node.go:183] Starting Egress Services Controller I0417 18:39:49.549871 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0417 18:39:49.549893 3681 shared_informer.go:356] "Caches are synced" controller="egressservices" I0417 18:39:49.549905 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0417 18:39:49.549914 3681 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0417 18:39:49.549925 3681 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0417 18:39:49.549936 3681 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0417 18:39:49.549944 3681 egressservice_node.go:197] Repairing Egress Services I0417 18:39:49.560149 3681 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0417 18:39:49.560163 3681 external_controller.go:278] Starting Admin Policy Based Route Controller I0417 18:39:49.560350 3681 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0417 18:39:49.560364 3681 link_network_manager.go:119] Link manager is running I0417 18:39:49.560375 3681 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0417 18:39:49.560495 3681 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0417 18:39:49.560777 3681 vrf_manager.go:128] VRF manager is running I0417 18:39:49.560891 3681 node_controller_manager.go:437] Removing flows to drop GARP I0417 18:39:49.560900 3681 gateway.go:532] Reconciling gateway with updates I0417 18:39:49.585640 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0417 18:39:49.585970 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0417 18:39:49.585980 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0417 18:39:49.585986 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0417 18:39:49.585985 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0417 18:39:49.585992 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0417 18:39:49.585997 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0417 18:39:49.585982 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0417 18:39:49.586007 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0417 18:39:49.586010 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0417 18:39:49.586006 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0417 18:39:49.586029 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0417 18:39:49.585653 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0417 18:39:49.585669 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0417 18:39:49.585992 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0417 18:39:49.586069 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0417 18:39:49.585707 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0417 18:39:49.586079 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0417 18:39:49.586083 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0417 18:39:49.586090 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0417 18:39:49.586099 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0417 18:39:49.586107 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0417 18:39:49.586123 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0417 18:39:49.585768 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0417 18:39:49.586154 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0417 18:39:49.586159 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0417 18:39:49.585751 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0417 18:39:49.586177 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0417 18:39:49.586179 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0417 18:39:49.585789 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0417 18:39:49.586201 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0417 18:39:49.586206 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0417 18:39:49.585797 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0417 18:39:49.586216 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0417 18:39:49.585778 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0417 18:39:49.586223 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0417 18:39:49.586228 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0417 18:39:49.585783 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0417 18:39:49.586239 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0417 18:39:49.586240 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0417 18:39:49.586261 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0417 18:39:49.585804 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0417 18:39:49.586280 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0417 18:39:49.586296 3681 port_claim.go:71] Svc openshift-ingress/router-default: port 31836 is already open, no action needed I0417 18:39:49.585812 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0417 18:39:49.586306 3681 port_claim.go:71] Svc openshift-ingress/router-default: port 32131 is already open, no action needed I0417 18:39:49.586310 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0417 18:39:49.586341 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0417 18:39:49.586217 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0417 18:39:49.586343 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0417 18:39:49.585852 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0417 18:39:49.586366 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0417 18:39:49.585854 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0417 18:39:49.586394 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0417 18:39:49.585866 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0417 18:39:49.586435 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0417 18:39:49.585868 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0417 18:39:49.586458 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0417 18:39:49.585874 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0417 18:39:49.586484 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0417 18:39:49.585871 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0417 18:39:49.586506 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0417 18:39:49.585882 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0417 18:39:49.586532 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0417 18:39:49.585891 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0417 18:39:49.586553 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0417 18:39:49.585893 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0417 18:39:49.586575 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0417 18:39:49.586579 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0417 18:39:49.585904 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0417 18:39:49.586586 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0417 18:39:49.585904 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0417 18:39:49.586609 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0417 18:39:49.586614 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0417 18:39:49.585907 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0417 18:39:49.586621 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0417 18:39:49.585916 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0417 18:39:49.586641 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0417 18:39:49.586644 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0417 18:39:49.585920 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0417 18:39:49.586651 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0417 18:39:49.585918 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0417 18:39:49.586706 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0417 18:39:49.585922 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0417 18:39:49.586732 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0417 18:39:49.586737 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0417 18:39:49.585924 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0417 18:39:49.586743 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0417 18:39:49.585959 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0417 18:39:49.586772 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0417 18:39:49.585961 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0417 18:39:49.586790 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0417 18:39:49.585985 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0417 18:39:49.585647 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0417 18:39:49.586915 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0417 18:39:49.585996 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0417 18:39:49.586002 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0417 18:39:49.586013 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0417 18:39:49.586038 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0417 18:39:49.586065 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0417 18:39:49.585673 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0417 18:39:49.587230 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0417 18:39:49.587235 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0417 18:39:49.586069 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0417 18:39:49.587241 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0417 18:39:49.585802 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0417 18:39:49.587247 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0417 18:39:49.587252 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0417 18:39:49.586271 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0417 18:39:49.585836 3681 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0417 18:39:49.587259 3681 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0417 18:39:49.626495 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0417 18:39:49.626520 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0417 18:39:49.626530 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0417 18:39:49.626538 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0417 18:39:49.626546 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0417 18:39:49.626553 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0417 18:39:49.666199 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0417 18:39:49.666225 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0417 18:39:49.666232 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0417 18:39:49.666252 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0417 18:39:49.666260 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0417 18:39:49.666279 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0417 18:39:49.666286 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0417 18:39:49.706290 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0417 18:39:49.706317 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0417 18:39:49.706325 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0417 18:39:49.706335 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0417 18:39:49.706344 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0417 18:39:49.706351 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0417 18:39:49.706365 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0417 18:39:49.706372 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0417 18:39:49.706380 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0417 18:39:49.706388 3681 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0417 18:39:51.496099 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-177.ec2.internal I0417 18:39:51.496136 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-177.ec2.internal: 44.74µs I0417 18:39:51.522755 3681 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-759bcd5579-sn6lt] creating logical port openshift-image-registry_image-registry-759bcd5579-sn6lt for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:51.522891 3681 kube.go:256] Updating pod openshift-image-registry/image-registry-759bcd5579-sn6lt I0417 18:39:51.540443 3681 pod.go:62] [openshift-image-registry/image-registry-759bcd5579-sn6lt] pod update took 17.587878ms I0417 18:39:51.540542 3681 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-759bcd5579-sn6lt] addLogicalPort annotation time took 17.69264ms I0417 18:39:51.542394 3681 pods.go:271] [openshift-image-registry/image-registry-759bcd5579-sn6lt] addLogicalPort took 19.652052ms, libovsdb time 1.486239ms I0417 18:39:51.570197 3681 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-gtn7z] creating logical port openshift-dns_dns-default-gtn7z for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:51.570339 3681 kube.go:256] Updating pod openshift-dns/dns-default-gtn7z I0417 18:39:51.572667 3681 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-h88sq] creating logical port openshift-ingress-canary_ingress-canary-h88sq for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:51.572826 3681 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-h88sq I0417 18:39:51.589941 3681 pod.go:62] [openshift-dns/dns-default-gtn7z] pod update took 19.63451ms I0417 18:39:51.589967 3681 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-gtn7z] addLogicalPort annotation time took 19.669475ms I0417 18:39:51.590005 3681 pod.go:62] [openshift-ingress-canary/ingress-canary-h88sq] pod update took 17.209516ms I0417 18:39:51.590026 3681 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-h88sq] addLogicalPort annotation time took 17.234246ms I0417 18:39:51.591324 3681 pods.go:271] [openshift-dns/dns-default-gtn7z] addLogicalPort took 21.173513ms, libovsdb time 967.683µs I0417 18:39:51.591782 3681 pods.go:271] [openshift-ingress-canary/ingress-canary-h88sq] addLogicalPort took 19.126869ms, libovsdb time 1.371651ms I0417 18:39:51.591854 3681 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-z8nvj] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-z8nvj for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:51.591921 3681 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-z8nvj I0417 18:39:51.608087 3681 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-z8nvj] pod update took 16.177948ms I0417 18:39:51.608115 3681 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-z8nvj] addLogicalPort annotation time took 16.213236ms I0417 18:39:51.609254 3681 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-z8nvj] addLogicalPort took 17.410429ms, libovsdb time 788.739µs I0417 18:39:51.978871 3681 cni.go:452] [openshift-ingress-canary/ingress-canary-h88sq 78fa73ec0fbfb4425095e84ff0e4927394469fa60d61c13905be0349d61661fb network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-h88sq 78fa73ec0fbfb4425095e84ff0e4927394469fa60d61c13905be0349d61661fb network default NAD default NAD key ] I0417 18:39:51.994308 3681 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-z8nvj 756c1b2a932d94aae5defe7f8ceb9985bdb9bde4cb4305df21771a3dfa594eeb network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-z8nvj 756c1b2a932d94aae5defe7f8ceb9985bdb9bde4cb4305df21771a3dfa594eeb network default NAD default NAD key ] I0417 18:39:52.019585 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-h88sq, hostIfaceName: 78fa73ec0fbfb44, network: default, NAD default, SandboxID: "78fa73ec0fbfb4425095e84ff0e4927394469fa60d61c13905be0349d61661fb", PCI device ID: , UID: "3f3930a3-385a-45d8-988e-bd645bf94ba2", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0417 18:39:52.019915 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-z8nvj, hostIfaceName: 756c1b2a932d94a, network: default, NAD default, SandboxID: "756c1b2a932d94aae5defe7f8ceb9985bdb9bde4cb4305df21771a3dfa594eeb", PCI device ID: , UID: "8074201d-7c83-4e86-847d-200bb11cf517", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0417 18:39:52.079055 3681 cni.go:473] [openshift-ingress-canary/ingress-canary-h88sq 78fa73ec0fbfb4425095e84ff0e4927394469fa60d61c13905be0349d61661fb network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-h88sq 78fa73ec0fbfb4425095e84ff0e4927394469fa60d61c13905be0349d61661fb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:aa:39:8b:57:ca\",\"name\":\"78fa73ec0fbfb44\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d012a6db-a993-4c6d-bd08-967c6fc059ba\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:39:52.090765 3681 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-z8nvj 756c1b2a932d94aae5defe7f8ceb9985bdb9bde4cb4305df21771a3dfa594eeb network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-z8nvj 756c1b2a932d94aae5defe7f8ceb9985bdb9bde4cb4305df21771a3dfa594eeb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:55:2e:18:37:5a\",\"name\":\"756c1b2a932d94a\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cdc59abe-fdfa-4c6f-9897-8a9d14150b4b\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:39:55.760797 3681 cni.go:452] [openshift-network-diagnostics/network-check-target-lkw5s e9b73e0533cb1cb3bc3a6d7dda31833b403563f64a2257d7329dc97215252bf2 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-lkw5s e9b73e0533cb1cb3bc3a6d7dda31833b403563f64a2257d7329dc97215252bf2 network default NAD default NAD key ] I0417 18:39:55.770619 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-lkw5s, hostIfaceName: e9b73e0533cb1cb, network: default, NAD default, SandboxID: "e9b73e0533cb1cb3bc3a6d7dda31833b403563f64a2257d7329dc97215252bf2", PCI device ID: , UID: "f9703f60-17fd-4c51-b027-05a2f003af66", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0417 18:39:55.819544 3681 cni.go:473] [openshift-network-diagnostics/network-check-target-lkw5s e9b73e0533cb1cb3bc3a6d7dda31833b403563f64a2257d7329dc97215252bf2 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-lkw5s e9b73e0533cb1cb3bc3a6d7dda31833b403563f64a2257d7329dc97215252bf2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:62:25:4f:57:12\",\"name\":\"e9b73e0533cb1cb\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4466180a-db39-427a-883e-b8d120ffdc11\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:39:56.579164 3681 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-9pppw] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-9pppw for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.579583 3681 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-9pppw I0417 18:39:56.596217 3681 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-9pppw] pod update took 16.638481ms I0417 18:39:56.596248 3681 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-9pppw] addLogicalPort annotation time took 16.693626ms I0417 18:39:56.598027 3681 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-9pppw] addLogicalPort took 18.885214ms, libovsdb time 1.286604ms I0417 18:39:56.683876 3681 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9b2jh] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-9b2jh for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.684019 3681 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9b2jh I0417 18:39:56.687924 3681 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-gndn4] creating logical port openshift-network-diagnostics_network-check-source-84fd6d9d54-gndn4 for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.687991 3681 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-w9b8h] creating logical port openshift-insights_insights-operator-6dbb575d6c-w9b8h for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.688079 3681 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-w9b8h I0417 18:39:56.688094 3681 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-84fd6d9d54-gndn4 I0417 18:39:56.700394 3681 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9b2jh] pod update took 16.413259ms I0417 18:39:56.700418 3681 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9b2jh] addLogicalPort annotation time took 16.443322ms I0417 18:39:56.701664 3681 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9b2jh] addLogicalPort took 17.809697ms, libovsdb time 910.319µs I0417 18:39:56.703507 3681 pod.go:62] [openshift-network-diagnostics/network-check-source-84fd6d9d54-gndn4] pod update took 15.464129ms I0417 18:39:56.703529 3681 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-gndn4] addLogicalPort annotation time took 15.491833ms I0417 18:39:56.704685 3681 pods.go:271] [openshift-network-diagnostics/network-check-source-84fd6d9d54-gndn4] addLogicalPort took 16.779458ms, libovsdb time 791.3µs I0417 18:39:56.706551 3681 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-w9b8h] pod update took 18.48943ms I0417 18:39:56.706574 3681 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-w9b8h] addLogicalPort annotation time took 18.5153ms I0417 18:39:56.708004 3681 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-w9b8h] addLogicalPort took 20.040861ms, libovsdb time 1.056484ms I0417 18:39:56.708068 3681 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-44nzf] creating logical port openshift-console-operator_console-operator-66dbbb6468-44nzf for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.708146 3681 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-44nzf I0417 18:39:56.720940 3681 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-44nzf] pod update took 12.810449ms I0417 18:39:56.720970 3681 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-44nzf] addLogicalPort annotation time took 12.845299ms I0417 18:39:56.722101 3681 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-44nzf] addLogicalPort took 14.045162ms, libovsdb time 833.738µs I0417 18:39:56.802848 3681 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-6825n] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-6825n for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.802964 3681 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-6825n I0417 18:39:56.811927 3681 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-l8xzw] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-l8xzw for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.812090 3681 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-l8xzw I0417 18:39:56.820976 3681 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-6825n] pod update took 18.037744ms I0417 18:39:56.821066 3681 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-6825n] addLogicalPort annotation time took 18.135255ms I0417 18:39:56.822662 3681 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-6825n] addLogicalPort took 19.83313ms, libovsdb time 1.209661ms I0417 18:39:56.836878 3681 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-l8xzw] pod update took 24.820423ms I0417 18:39:56.836915 3681 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-l8xzw] addLogicalPort annotation time took 24.865241ms I0417 18:39:56.837032 3681 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-759fbb4fcf-xt7hr] creating logical port openshift-ingress_router-default-759fbb4fcf-xt7hr for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.837180 3681 kube.go:256] Updating pod openshift-ingress/router-default-759fbb4fcf-xt7hr I0417 18:39:56.837264 3681 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6p7fs] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-6p7fs for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.837334 3681 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6p7fs I0417 18:39:56.838325 3681 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-l8xzw] addLogicalPort took 26.42114ms, libovsdb time 1.019936ms I0417 18:39:56.838375 3681 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-vbfj2] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-vbfj2 for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.838426 3681 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-vbfj2 I0417 18:39:56.853848 3681 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6p7fs] pod update took 16.531373ms I0417 18:39:56.853875 3681 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6p7fs] addLogicalPort annotation time took 16.560692ms I0417 18:39:56.855328 3681 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6p7fs] addLogicalPort took 18.102054ms, libovsdb time 1.045909ms I0417 18:39:56.856788 3681 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-vbfj2] pod update took 18.373112ms I0417 18:39:56.856806 3681 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-vbfj2] addLogicalPort annotation time took 18.396593ms I0417 18:39:56.858199 3681 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-vbfj2] addLogicalPort took 19.826197ms, libovsdb time 1.054455ms I0417 18:39:56.860490 3681 pod.go:62] [openshift-ingress/router-default-759fbb4fcf-xt7hr] pod update took 23.346855ms I0417 18:39:56.860515 3681 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-759fbb4fcf-xt7hr] addLogicalPort annotation time took 23.375585ms I0417 18:39:56.860708 3681 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-pdcx6] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-pdcx6 for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.860853 3681 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-pdcx6 I0417 18:39:56.861939 3681 pods.go:271] [openshift-ingress/router-default-759fbb4fcf-xt7hr] addLogicalPort took 24.926972ms, libovsdb time 964.363µs I0417 18:39:56.872565 3681 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-pdcx6] pod update took 11.744805ms I0417 18:39:56.872586 3681 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-pdcx6] addLogicalPort annotation time took 11.77228ms I0417 18:39:56.873781 3681 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-pdcx6] addLogicalPort took 13.107991ms, libovsdb time 820.916µs I0417 18:39:56.902142 3681 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7797cc64cd-5kqr8] creating logical port openshift-ingress_router-default-7797cc64cd-5kqr8 for pod on switch ip-10-0-2-177.ec2.internal I0417 18:39:56.902244 3681 kube.go:256] Updating pod openshift-ingress/router-default-7797cc64cd-5kqr8 I0417 18:39:56.920207 3681 pod.go:62] [openshift-ingress/router-default-7797cc64cd-5kqr8] pod update took 17.987274ms I0417 18:39:56.920236 3681 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-7797cc64cd-5kqr8] addLogicalPort annotation time took 18.023021ms I0417 18:39:56.921670 3681 pods.go:271] [openshift-ingress/router-default-7797cc64cd-5kqr8] addLogicalPort took 19.548359ms, libovsdb time 1.069133ms I0417 18:39:57.176769 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-68.ec2.internal in network "default" I0417 18:39:57.176792 3681 node_tracker.go:220] Node ip-10-0-1-68.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0417 18:39:57.176776 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-68.ec2.internal I0417 18:39:57.176811 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-68.ec2.internal: 39.308µs I0417 18:39:57.176821 3681 egressfirewall.go:723] Syncing node "ip-10-0-1-68.ec2.internal" for egress firewall E0417 18:39:57.176837 3681 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-68.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0417 18:39:57.176856 3681 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-68.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-68.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-68.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-68.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-68.ec2.internal, error parsing annotation for node ip-10-0-1-68.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0417 18:39:57.192063 3681 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-68.ec2.internal annotation: unexpected end of JSON input E0417 18:39:57.202852 3681 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-68.ec2.internal annotation: unexpected end of JSON input I0417 18:39:57.202876 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0417 18:39:57.202927 3681 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-68.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-68.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-68.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-68.ec2.internal" I0417 18:39:57.210943 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0417 18:39:57.211018 3681 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-68.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-68.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-68.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-68.ec2.internal" I0417 18:39:57.226329 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-68.ec2.internal I0417 18:39:57.226351 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-68.ec2.internal: 30.515µs I0417 18:39:57.226418 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:39:57.226428 3681 egressfirewall.go:723] Syncing node "ip-10-0-1-68.ec2.internal" for egress firewall E0417 18:39:57.226486 3681 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-68.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-1-68.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-1-68.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-1-68.ec2.internal" I0417 18:39:57.252104 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0417 18:39:57.436198 3681 egressfirewall.go:723] Syncing node "ip-10-0-1-68.ec2.internal" for egress firewall I0417 18:39:57.436209 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-68.ec2.internal I0417 18:39:57.436242 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-68.ec2.internal: 36.915µs I0417 18:39:57.436286 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:39:57.451915 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:39:57.638819 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:39:57.701928 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:39:57.743336 3681 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-44nzf ad810d5bf65667378d2a13d7e3a8faa29e0428e9e241edf5b5288fe1969fa542 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-44nzf ad810d5bf65667378d2a13d7e3a8faa29e0428e9e241edf5b5288fe1969fa542 network default NAD default NAD key ] I0417 18:39:57.753909 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-44nzf, hostIfaceName: ad810d5bf656673, network: default, NAD default, SandboxID: "ad810d5bf65667378d2a13d7e3a8faa29e0428e9e241edf5b5288fe1969fa542", PCI device ID: , UID: "e523969d-ec24-4816-8fc5-c819372c434b", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0417 18:39:57.810546 3681 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-44nzf ad810d5bf65667378d2a13d7e3a8faa29e0428e9e241edf5b5288fe1969fa542 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-44nzf ad810d5bf65667378d2a13d7e3a8faa29e0428e9e241edf5b5288fe1969fa542 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:7e:37:d6:ea:58\",\"name\":\"ad810d5bf656673\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4daa935-903c-4b8f-86ae-3c94ce85e6a2\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:39:57.922827 3681 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-w9b8h 7e5a346dea86d9ecea6930a1dec6e578ae880bb59a14186e640e41334c9373a7 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-w9b8h 7e5a346dea86d9ecea6930a1dec6e578ae880bb59a14186e640e41334c9373a7 network default NAD default NAD key ] I0417 18:39:57.922829 3681 cni.go:452] [openshift-network-diagnostics/network-check-source-84fd6d9d54-gndn4 b04d3be9782af13df5c8bd0f7a42fcf832e12bc300f90523d9e17fe65fe9b3d9 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-gndn4 b04d3be9782af13df5c8bd0f7a42fcf832e12bc300f90523d9e17fe65fe9b3d9 network default NAD default NAD key ] I0417 18:39:57.923900 3681 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6p7fs bb2a11b977403d4f0cf019c53de98099a32c635022497667de73da41bfc3289e network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6p7fs bb2a11b977403d4f0cf019c53de98099a32c635022497667de73da41bfc3289e network default NAD default NAD key ] I0417 18:39:57.926004 3681 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-vbfj2 b903c37fa08853099e135d2b72cb05e5746adff8ad071627d61508f37019c9da network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-vbfj2 b903c37fa08853099e135d2b72cb05e5746adff8ad071627d61508f37019c9da network default NAD default NAD key ] I0417 18:39:57.948153 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-w9b8h, hostIfaceName: 7e5a346dea86d9e, network: default, NAD default, SandboxID: "7e5a346dea86d9ecea6930a1dec6e578ae880bb59a14186e640e41334c9373a7", PCI device ID: , UID: "88532c56-c4c1-49fe-9d03-984b857cbe82", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0417 18:39:57.948153 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-84fd6d9d54-gndn4, hostIfaceName: b04d3be9782af13, network: default, NAD default, SandboxID: "b04d3be9782af13df5c8bd0f7a42fcf832e12bc300f90523d9e17fe65fe9b3d9", PCI device ID: , UID: "b5c3e824-eb63-4125-b823-3f98321317e7", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0417 18:39:57.955554 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-vbfj2, hostIfaceName: b903c37fa088530, network: default, NAD default, SandboxID: "b903c37fa08853099e135d2b72cb05e5746adff8ad071627d61508f37019c9da", PCI device ID: , UID: "b436bc6b-09a0-4ec5-aeb6-059a65467c61", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0417 18:39:57.956667 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-6p7fs, hostIfaceName: bb2a11b977403d4, network: default, NAD default, SandboxID: "bb2a11b977403d4f0cf019c53de98099a32c635022497667de73da41bfc3289e", PCI device ID: , UID: "c0704916-04f7-46bb-b262-afa3e14bcb89", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0417 18:39:58.004290 3681 cni.go:473] [openshift-network-diagnostics/network-check-source-84fd6d9d54-gndn4 b04d3be9782af13df5c8bd0f7a42fcf832e12bc300f90523d9e17fe65fe9b3d9 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-gndn4 b04d3be9782af13df5c8bd0f7a42fcf832e12bc300f90523d9e17fe65fe9b3d9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:6e:cf:51:cf:ff\",\"name\":\"b04d3be9782af13\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/357ea83c-65fc-4276-a764-e8b3b2652923\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:39:58.135536 3681 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9b2jh 9107718c03eed4cddb731b5bcd5bd8de8a6673e670f967e0a0e58175bee9059a network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9b2jh 9107718c03eed4cddb731b5bcd5bd8de8a6673e670f967e0a0e58175bee9059a network default NAD default NAD key ] I0417 18:39:58.147414 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-9b2jh, hostIfaceName: 9107718c03eed4c, network: default, NAD default, SandboxID: "9107718c03eed4cddb731b5bcd5bd8de8a6673e670f967e0a0e58175bee9059a", PCI device ID: , UID: "15c9d9fd-14ee-4bdc-a3e2-8776b415beb4", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0417 18:39:58.199714 3681 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9b2jh 9107718c03eed4cddb731b5bcd5bd8de8a6673e670f967e0a0e58175bee9059a network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-9b2jh 9107718c03eed4cddb731b5bcd5bd8de8a6673e670f967e0a0e58175bee9059a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:3a:d9:42:db:08\",\"name\":\"9107718c03eed4c\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b21e198a-aa17-44e9-88f7-54101c93a40a\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:39:58.230232 3681 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-w9b8h 7e5a346dea86d9ecea6930a1dec6e578ae880bb59a14186e640e41334c9373a7 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-w9b8h 7e5a346dea86d9ecea6930a1dec6e578ae880bb59a14186e640e41334c9373a7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:f1:80:30:b3:98\",\"name\":\"7e5a346dea86d9e\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e7cb6f4a-fab1-49de-99d6-50d1398fcb68\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:39:58.230317 3681 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6p7fs bb2a11b977403d4f0cf019c53de98099a32c635022497667de73da41bfc3289e network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-6p7fs bb2a11b977403d4f0cf019c53de98099a32c635022497667de73da41bfc3289e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:cb:9c:76:c3:0a\",\"name\":\"bb2a11b977403d4\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a1e38ce8-5af8-444e-8cc9-861108395f73\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:39:58.230323 3681 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-vbfj2 b903c37fa08853099e135d2b72cb05e5746adff8ad071627d61508f37019c9da network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-vbfj2 b903c37fa08853099e135d2b72cb05e5746adff8ad071627d61508f37019c9da network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:ac:f2:74:2b:9b\",\"name\":\"b903c37fa088530\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4c773fba-c404-495f-a028-3d5ef606df78\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:39:59.714767 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-216.ec2.internal in network "default" I0417 18:39:59.714803 3681 node_tracker.go:220] Node ip-10-0-0-216.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0417 18:39:59.714818 3681 egressfirewall.go:723] Syncing node "ip-10-0-0-216.ec2.internal" for egress firewall I0417 18:39:59.714837 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-216.ec2.internal I0417 18:39:59.714861 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-216.ec2.internal: 29.495µs E0417 18:39:59.714924 3681 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-216.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0417 18:39:59.714943 3681 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-216.ec2.internal, error: [adding or updating remote node chassis ip-10-0-0-216.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-216.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-216.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-216.ec2.internal, error parsing annotation for node ip-10-0-0-216.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] E0417 18:39:59.729176 3681 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-216.ec2.internal annotation: unexpected end of JSON input I0417 18:39:59.729239 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0417 18:39:59.729274 3681 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-216.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0417 18:39:59.745514 3681 egressfirewall.go:723] Syncing node "ip-10-0-0-216.ec2.internal" for egress firewall I0417 18:39:59.745539 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-216.ec2.internal I0417 18:39:59.745560 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-216.ec2.internal: 27.602µs E0417 18:39:59.745598 3681 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-216.ec2.internal annotation: unexpected end of JSON input I0417 18:39:59.745621 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0417 18:39:59.745655 3681 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-216.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0417 18:39:59.824557 3681 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-216.ec2.internal annotation: unexpected end of JSON input E0417 18:39:59.824582 3681 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-0-216.ec2.internal annotation: unexpected end of JSON input I0417 18:39:59.824594 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0417 18:39:59.824673 3681 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-0-216.ec2.internal: suppressed error logged: failed to fallback to annotations after error "failed to generate gateway router port address for node ip-10-0-0-216.ec2.internal: k8s.ovn.org/node-id annotation not found for node ip-10-0-0-216.ec2.internal": k8s.ovn.org/node-gateway-router-lrp-ifaddr annotation not found for node "ip-10-0-0-216.ec2.internal" I0417 18:39:59.841789 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true I0417 18:39:59.961712 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:39:59.961762 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-216.ec2.internal I0417 18:39:59.961777 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-216.ec2.internal: 19.278µs I0417 18:39:59.961763 3681 egressfirewall.go:723] Syncing node "ip-10-0-0-216.ec2.internal" for egress firewall I0417 18:39:59.979822 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:00.044482 3681 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-initial-4g8fb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:40:00.044531 3681 obj_retry.go:498] Detected object kube-system/ecr-credential-refresh-initial-4g8fb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:40:00.191650 3681 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29607520-4b58g for pod on switch ip-10-0-2-177.ec2.internal I0417 18:40:00.191801 3681 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g I0417 18:40:00.203630 3681 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g] pod update took 11.853087ms I0417 18:40:00.203656 3681 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g] addLogicalPort annotation time took 11.88598ms I0417 18:40:00.204945 3681 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g] addLogicalPort took 13.317382ms, libovsdb time 923.261µs I0417 18:40:00.225167 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:00.886335 3681 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g 5701fae4858326300d3d95517c528fa6dea982c8a52c4bb4a2e3eb2226768323 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g 5701fae4858326300d3d95517c528fa6dea982c8a52c4bb4a2e3eb2226768323 network default NAD default NAD key ] I0417 18:40:00.900613 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29607520-4b58g, hostIfaceName: 5701fae48583263, network: default, NAD default, SandboxID: "5701fae4858326300d3d95517c528fa6dea982c8a52c4bb4a2e3eb2226768323", PCI device ID: , UID: "3465975f-6586-4075-afc7-c282745230a1", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0417 18:40:00.968758 3681 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g 5701fae4858326300d3d95517c528fa6dea982c8a52c4bb4a2e3eb2226768323 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g 5701fae4858326300d3d95517c528fa6dea982c8a52c4bb4a2e3eb2226768323 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:7f:42:29:f5:43\",\"name\":\"5701fae48583263\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe26753f-9322-47d6-8f37-2a52613099cf\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:01.069340 3681 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-initial-4g8fb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:40:01.069444 3681 obj_retry.go:492] Detected object kube-system/ecr-credential-refresh-initial-4g8fb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0417 18:40:01.908190 3681 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0417 18:40:01.958025 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0417 18:40:02.572196 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0417 18:40:04.898220 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0417 18:40:05.569610 3681 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0417 18:40:05.570581 3681 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 948.717µs I0417 18:40:05.586563 3681 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0417 18:40:05.694822 3681 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zzwdf] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-zzwdf for pod on switch ip-10-0-2-177.ec2.internal I0417 18:40:05.694925 3681 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zzwdf I0417 18:40:05.710021 3681 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zzwdf] pod update took 15.117607ms I0417 18:40:05.710052 3681 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zzwdf] addLogicalPort annotation time took 15.155175ms I0417 18:40:05.711276 3681 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zzwdf] addLogicalPort took 16.468714ms, libovsdb time 876.703µs I0417 18:40:06.068572 3681 namespace.go:98] [openshift-service-ca] adding namespace I0417 18:40:06.069455 3681 namespace.go:102] [openshift-service-ca] adding namespace took 860.081µs I0417 18:40:06.085629 3681 namespace.go:142] [openshift-service-ca] updating namespace I0417 18:40:06.101832 3681 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zzwdf 64914ef34b13cdb95194c416e9440f6e5d09f39ded19478c3f86bc73d08abe55 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zzwdf 64914ef34b13cdb95194c416e9440f6e5d09f39ded19478c3f86bc73d08abe55 network default NAD default NAD key ] I0417 18:40:06.116557 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-zzwdf, hostIfaceName: 64914ef34b13cdb, network: default, NAD default, SandboxID: "64914ef34b13cdb95194c416e9440f6e5d09f39ded19478c3f86bc73d08abe55", PCI device ID: , UID: "951fe313-692b-4faf-9e09-176510b0d4e9", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0417 18:40:06.169836 3681 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zzwdf 64914ef34b13cdb95194c416e9440f6e5d09f39ded19478c3f86bc73d08abe55 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-zzwdf 64914ef34b13cdb95194c416e9440f6e5d09f39ded19478c3f86bc73d08abe55 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:ad:8f:3b:98:49\",\"name\":\"64914ef34b13cdb\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9960f316-c397-451b-9c4d-ab3eb9f63388\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:07.523109 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:07.748779 3681 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-mwv67] creating logical port openshift-service-ca_service-ca-d9cb48497-mwv67 for pod on switch ip-10-0-2-177.ec2.internal I0417 18:40:07.748888 3681 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-mwv67 I0417 18:40:07.761679 3681 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-mwv67] pod update took 12.813621ms I0417 18:40:07.761748 3681 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-mwv67] addLogicalPort annotation time took 12.888191ms I0417 18:40:07.763152 3681 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-mwv67] addLogicalPort took 14.396861ms, libovsdb time 989.15µs I0417 18:40:08.152501 3681 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-mwv67 9755eca973a2d13efc6de1041305d1aed0df34aea8742652c9b061267b125362 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-mwv67 9755eca973a2d13efc6de1041305d1aed0df34aea8742652c9b061267b125362 network default NAD default NAD key ] I0417 18:40:08.162253 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-mwv67, hostIfaceName: 9755eca973a2d13, network: default, NAD default, SandboxID: "9755eca973a2d13efc6de1041305d1aed0df34aea8742652c9b061267b125362", PCI device ID: , UID: "4d6bb0d2-d821-446f-a265-cad797f242f6", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0417 18:40:08.212762 3681 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-mwv67 9755eca973a2d13efc6de1041305d1aed0df34aea8742652c9b061267b125362 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-mwv67 9755eca973a2d13efc6de1041305d1aed0df34aea8742652c9b061267b125362 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:34:e6:e6:16:69\",\"name\":\"9755eca973a2d13\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2975185-b95d-4ee8-ac38-9a273c4581d2\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:09.985085 3681 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-d8n5h] creating logical port openshift-insights_insights-runtime-extractor-d8n5h for pod on switch ip-10-0-2-177.ec2.internal I0417 18:40:09.985198 3681 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-d8n5h I0417 18:40:10.005728 3681 pod.go:62] [openshift-insights/insights-runtime-extractor-d8n5h] pod update took 20.560977ms I0417 18:40:10.005762 3681 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-d8n5h] addLogicalPort annotation time took 20.600768ms I0417 18:40:10.007419 3681 pods.go:271] [openshift-insights/insights-runtime-extractor-d8n5h] addLogicalPort took 22.354763ms, libovsdb time 1.155816ms I0417 18:40:10.065814 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:12.891297 3681 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-9pppw 1a728336076321e6a0b3634d7c89e850628b1bfc5f685c217682f82ad1f7256f network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-9pppw 1a728336076321e6a0b3634d7c89e850628b1bfc5f685c217682f82ad1f7256f network default NAD default NAD key ] I0417 18:40:12.901071 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-9pppw, hostIfaceName: 1a728336076321e, network: default, NAD default, SandboxID: "1a728336076321e6a0b3634d7c89e850628b1bfc5f685c217682f82ad1f7256f", PCI device ID: , UID: "a6b0f46f-0269-4826-80a0-12fd1655d650", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0417 18:40:12.951832 3681 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-9pppw 1a728336076321e6a0b3634d7c89e850628b1bfc5f685c217682f82ad1f7256f network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-9pppw 1a728336076321e6a0b3634d7c89e850628b1bfc5f685c217682f82ad1f7256f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:58:c4:c8:88:7e\",\"name\":\"1a728336076321e\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7538d3cb-82ff-4d70-b30e-0281ec3f5cc2\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:40:13.638405 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0417 18:40:15.656740 3681 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-0-216.ec2.internal I0417 18:40:15.656783 3681 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-0-216.ec2.internal I0417 18:40:15.656746 3681 obj_retry.go:334] Retry object setup: *v1.Node ip-10-0-1-68.ec2.internal I0417 18:40:15.656860 3681 obj_retry.go:398] Adding new object: *v1.Node ip-10-0-1-68.ec2.internal I0417 18:40:15.656894 3681 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-0-216.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-0-216.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-216.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-216.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-216.ec2.internal I0417 18:40:15.656943 3681 obj_retry.go:409] Retry add failed for *v1.Node ip-10-0-1-68.ec2.internal, will try again later: adding or updating remote node chassis ip-10-0-1-68.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-68.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-68.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-68.ec2.internal I0417 18:40:17.100930 3681 egressfirewall.go:723] Syncing node "ip-10-0-1-68.ec2.internal" for egress firewall I0417 18:40:17.100925 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:17.100959 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-68.ec2.internal I0417 18:40:17.100974 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-68.ec2.internal: 19.54µs I0417 18:40:17.641914 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:17.641969 3681 egressfirewall.go:723] Syncing node "ip-10-0-0-216.ec2.internal" for egress firewall I0417 18:40:17.641996 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-216.ec2.internal I0417 18:40:17.642011 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-216.ec2.internal: 19.501µs I0417 18:40:18.052074 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-2wm8h I0417 18:40:18.052081 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-2m2dc I0417 18:40:18.052104 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-2wm8h I0417 18:40:18.052104 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-dsqzr I0417 18:40:18.052121 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-dsqzr I0417 18:40:18.052158 3681 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-2wm8h, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-2wm8h: suppressed error logged: pod openshift-multus/network-metrics-daemon-2wm8h: no pod IPs found I0417 18:40:18.052111 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-2m2dc I0417 18:40:18.052158 3681 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-dsqzr, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-dsqzr: suppressed error logged: pod openshift-network-diagnostics/network-check-target-dsqzr: no pod IPs found I0417 18:40:18.052182 3681 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-2m2dc, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-2m2dc: suppressed error logged: pod openshift-network-diagnostics/network-check-target-2m2dc: no pod IPs found I0417 18:40:18.052092 3681 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-6mcf6 I0417 18:40:18.052196 3681 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-6mcf6 I0417 18:40:18.052213 3681 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-6mcf6, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-6mcf6: suppressed error logged: pod openshift-multus/network-metrics-daemon-6mcf6: no pod IPs found I0417 18:40:23.707561 3681 cni.go:452] [openshift-image-registry/image-registry-759bcd5579-sn6lt 519395bc8cd67261dd0d93346924cc62372f584b467a23228bfecd9449ff3073 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-759bcd5579-sn6lt 519395bc8cd67261dd0d93346924cc62372f584b467a23228bfecd9449ff3073 network default NAD default NAD key ] I0417 18:40:23.717163 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-759bcd5579-sn6lt, hostIfaceName: 519395bc8cd6726, network: default, NAD default, SandboxID: "519395bc8cd67261dd0d93346924cc62372f584b467a23228bfecd9449ff3073", PCI device ID: , UID: "a4882604-dc16-4970-b431-8f025a87c1f0", MAC: 0a:58:0a:80:00:05, IPs: [10.128.0.5/23] I0417 18:40:23.750773 3681 cni.go:452] [openshift-dns/dns-default-gtn7z 184648bbe5135368c09f49e7312543aeb9bcfcbb43373b5c0eff685dd11cef10 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-gtn7z 184648bbe5135368c09f49e7312543aeb9bcfcbb43373b5c0eff685dd11cef10 network default NAD default NAD key ] I0417 18:40:23.764867 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-gtn7z, hostIfaceName: 184648bbe513536, network: default, NAD default, SandboxID: "184648bbe5135368c09f49e7312543aeb9bcfcbb43373b5c0eff685dd11cef10", PCI device ID: , UID: "e7c1a894-81b4-4636-a241-60870d944877", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0417 18:40:23.770671 3681 cni.go:473] [openshift-image-registry/image-registry-759bcd5579-sn6lt 519395bc8cd67261dd0d93346924cc62372f584b467a23228bfecd9449ff3073 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-759bcd5579-sn6lt 519395bc8cd67261dd0d93346924cc62372f584b467a23228bfecd9449ff3073 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:55:50:7f:ca:77\",\"name\":\"519395bc8cd6726\"},{\"mac\":\"0a:58:0a:80:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1da1d045-df10-4bb3-8e1d-8764b6ec786a\"}],\"ips\":[{\"address\":\"10.128.0.5/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:23.815095 3681 cni.go:473] [openshift-dns/dns-default-gtn7z 184648bbe5135368c09f49e7312543aeb9bcfcbb43373b5c0eff685dd11cef10 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-gtn7z 184648bbe5135368c09f49e7312543aeb9bcfcbb43373b5c0eff685dd11cef10 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:ed:64:8a:7e:96\",\"name\":\"184648bbe513536\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c1842d24-fe3f-4fa1-a1c5-6757b613fd7e\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:25.968948 3681 cni.go:452] [openshift-insights/insights-runtime-extractor-d8n5h b64c396487e3c62dc512cdfa8b498194b81349d9b5e5cc84adc39553a87f0b68 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-d8n5h b64c396487e3c62dc512cdfa8b498194b81349d9b5e5cc84adc39553a87f0b68 network default NAD default NAD key ] I0417 18:40:25.979287 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-d8n5h, hostIfaceName: b64c396487e3c62, network: default, NAD default, SandboxID: "b64c396487e3c62dc512cdfa8b498194b81349d9b5e5cc84adc39553a87f0b68", PCI device ID: , UID: "b2ea718b-ea52-475d-bb3a-d9ef99bfe88d", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0417 18:40:26.031778 3681 cni.go:473] [openshift-insights/insights-runtime-extractor-d8n5h b64c396487e3c62dc512cdfa8b498194b81349d9b5e5cc84adc39553a87f0b68 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-d8n5h b64c396487e3c62dc512cdfa8b498194b81349d9b5e5cc84adc39553a87f0b68 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:e6:71:5c:b6:22\",\"name\":\"b64c396487e3c62\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/70d1f3b9-b3e4-48d7-99ec-1077a6f5dd10\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:27.496997 3681 cni.go:452] [openshift-multus/network-metrics-daemon-lp6tg 3f7f3286ff88ee0cfeeb6c18b70fd1b7280464ab33ddfe4c280a8b6085eb432a network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-lp6tg 3f7f3286ff88ee0cfeeb6c18b70fd1b7280464ab33ddfe4c280a8b6085eb432a network default NAD default NAD key ] I0417 18:40:27.506636 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-lp6tg, hostIfaceName: 3f7f3286ff88ee0, network: default, NAD default, SandboxID: "3f7f3286ff88ee0cfeeb6c18b70fd1b7280464ab33ddfe4c280a8b6085eb432a", PCI device ID: , UID: "161e5395-4060-4c73-8321-ff4cb2210d9b", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0417 18:40:27.557382 3681 cni.go:473] [openshift-multus/network-metrics-daemon-lp6tg 3f7f3286ff88ee0cfeeb6c18b70fd1b7280464ab33ddfe4c280a8b6085eb432a network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-lp6tg 3f7f3286ff88ee0cfeeb6c18b70fd1b7280464ab33ddfe4c280a8b6085eb432a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:06:04:58:55:c0\",\"name\":\"3f7f3286ff88ee0\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4c54deec-3fc8-4bca-a57e-c5a93fb33c45\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:28.077768 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:29.323644 3681 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-6825n e8e544ece5adbcc03e39ae71cbd47ed4b45300f213dc209180f2c6f091fae5ed network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-6825n e8e544ece5adbcc03e39ae71cbd47ed4b45300f213dc209180f2c6f091fae5ed network default NAD default NAD key ] I0417 18:40:29.334478 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-6825n, hostIfaceName: e8e544ece5adbcc, network: default, NAD default, SandboxID: "e8e544ece5adbcc03e39ae71cbd47ed4b45300f213dc209180f2c6f091fae5ed", PCI device ID: , UID: "cbf0879a-ccba-4e35-ae40-c72ebeada169", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0417 18:40:29.350225 3681 cni.go:452] [openshift-ingress/router-default-759fbb4fcf-xt7hr d3420907029eda2f108ba93ed7f69741df979fce0c22db961ec8aa28ad4e85de network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-759fbb4fcf-xt7hr d3420907029eda2f108ba93ed7f69741df979fce0c22db961ec8aa28ad4e85de network default NAD default NAD key ] I0417 18:40:29.363385 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-759fbb4fcf-xt7hr, hostIfaceName: d3420907029eda2, network: default, NAD default, SandboxID: "d3420907029eda2f108ba93ed7f69741df979fce0c22db961ec8aa28ad4e85de", PCI device ID: , UID: "ea808efa-0e23-4a85-9a8a-e2739b6b03b9", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0417 18:40:29.363506 3681 cni.go:452] [openshift-ingress/router-default-7797cc64cd-5kqr8 5034bde48f37953adf26a93f45bcdd9bf7ba9e89301b475265202fe0728da3d9 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-7797cc64cd-5kqr8 5034bde48f37953adf26a93f45bcdd9bf7ba9e89301b475265202fe0728da3d9 network default NAD default NAD key ] I0417 18:40:29.363506 3681 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-l8xzw 8cb2316c8e7ed6f867106921b2ddaded90be1f5ecc310d148361024f2e058a78 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-l8xzw 8cb2316c8e7ed6f867106921b2ddaded90be1f5ecc310d148361024f2e058a78 network default NAD default NAD key ] I0417 18:40:29.363940 3681 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-pdcx6 aa40e53f84a1ed457d9e153317b1cfc431eff58c27d80608cdba083d91f6019b network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-pdcx6 aa40e53f84a1ed457d9e153317b1cfc431eff58c27d80608cdba083d91f6019b network default NAD default NAD key ] I0417 18:40:29.382086 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-l8xzw, hostIfaceName: 8cb2316c8e7ed6f, network: default, NAD default, SandboxID: "8cb2316c8e7ed6f867106921b2ddaded90be1f5ecc310d148361024f2e058a78", PCI device ID: , UID: "1202a7a7-7039-447e-8893-f05c8914ae69", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0417 18:40:29.386889 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-pdcx6, hostIfaceName: aa40e53f84a1ed4, network: default, NAD default, SandboxID: "aa40e53f84a1ed457d9e153317b1cfc431eff58c27d80608cdba083d91f6019b", PCI device ID: , UID: "dedeb88d-5b34-43a3-b994-73c5a1bf44ad", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0417 18:40:29.387815 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7797cc64cd-5kqr8, hostIfaceName: 5034bde48f37953, network: default, NAD default, SandboxID: "5034bde48f37953adf26a93f45bcdd9bf7ba9e89301b475265202fe0728da3d9", PCI device ID: , UID: "0550565d-7057-441b-a441-17bc065fa05f", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0417 18:40:29.413361 3681 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-6825n e8e544ece5adbcc03e39ae71cbd47ed4b45300f213dc209180f2c6f091fae5ed network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-6825n e8e544ece5adbcc03e39ae71cbd47ed4b45300f213dc209180f2c6f091fae5ed network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:85:c6:bc:c1:8c\",\"name\":\"e8e544ece5adbcc\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c3c3fd95-8a9e-4a41-9506-2f4f91150a35\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:29.442356 3681 cni.go:473] [openshift-ingress/router-default-759fbb4fcf-xt7hr d3420907029eda2f108ba93ed7f69741df979fce0c22db961ec8aa28ad4e85de network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-759fbb4fcf-xt7hr d3420907029eda2f108ba93ed7f69741df979fce0c22db961ec8aa28ad4e85de network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:0e:39:21:aa:61\",\"name\":\"d3420907029eda2\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f38e2cc1-a567-40d0-8de3-537a81044b3f\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:40:29.534310 3681 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0417 18:40:29.539779 3681 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:29.539797 3681 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:29.539802 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:29.549328 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0417 18:40:29.561010 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:29.600054 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0417 18:40:29.632572 3681 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-6dc58774f6-v5hvv IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-6dc58774f6-v5hvv: no pod IPs found W0417 18:40:29.651102 3681 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-759bcd5579-cvhl5 IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-759bcd5579-cvhl5: no pod IPs found I0417 18:40:29.672099 3681 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-l8xzw 8cb2316c8e7ed6f867106921b2ddaded90be1f5ecc310d148361024f2e058a78 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-l8xzw 8cb2316c8e7ed6f867106921b2ddaded90be1f5ecc310d148361024f2e058a78 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:79:13:fa:86:ef\",\"name\":\"8cb2316c8e7ed6f\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f475f967-85c7-48aa-b3ba-9c0f02520356\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:29.672105 3681 cni.go:473] [openshift-ingress/router-default-7797cc64cd-5kqr8 5034bde48f37953adf26a93f45bcdd9bf7ba9e89301b475265202fe0728da3d9 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-7797cc64cd-5kqr8 5034bde48f37953adf26a93f45bcdd9bf7ba9e89301b475265202fe0728da3d9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:3e:48:c4:03:63\",\"name\":\"5034bde48f37953\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8dbd84b5-bdda-4b49-9f33-d3f89490e9f4\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:29.672097 3681 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-pdcx6 aa40e53f84a1ed457d9e153317b1cfc431eff58c27d80608cdba083d91f6019b network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-pdcx6 aa40e53f84a1ed457d9e153317b1cfc431eff58c27d80608cdba083d91f6019b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:12:b1:88:4a:83\",\"name\":\"aa40e53f84a1ed4\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6ae9a0a8-0679-40e5-a12d-2a5e0e968b1b\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:29.704881 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:29.732968 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:29.915441 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-68.ec2.internal in network "default" I0417 18:40:29.915504 3681 node_tracker.go:235] Node ip-10-0-1-68.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-68.ec2.internal" I0417 18:40:29.915527 3681 node_tracker.go:169] Node ip-10-0-1-68.ec2.internal switch + router changed, syncing services in network "default" I0417 18:40:29.915539 3681 services_controller.go:574] Full service sync requested for network=default I0417 18:40:29.915546 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "ip-10-0-1-68.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0417 18:40:29.915780 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0417 18:40:29.915817 3681 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:29.915835 3681 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:29.915841 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:29.915859 3681 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0417 18:40:29.916018 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0417 18:40:29.973582 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "ip-10-0-1-68.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:30.064610 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-68.ec2.internal in network "default" I0417 18:40:30.064665 3681 default_network_controller.go:1025] Node "ip-10-0-1-68.ec2.internal" in remote zone "ip-10-0-1-68.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:30.064675 3681 node_tracker.go:169] Node ip-10-0-1-68.ec2.internal switch + router changed, syncing services in network "default" I0417 18:40:30.064711 3681 services_controller.go:574] Full service sync requested for network=default W0417 18:40:30.064969 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0417 18:40:30.065021 3681 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.065036 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.065047 3681 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.065072 3681 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0417 18:40:30.065297 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0417 18:40:30.065684 3681 zone_ic_handler.go:195] Time taken to create transit switch: 276.857µs I0417 18:40:30.065716 3681 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-68.ec2.internal for the network default I0417 18:40:30.066709 3681 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-68.ec2.internal" on network "default" took: 1.293806ms W0417 18:40:30.195941 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.195964 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.204994 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.205012 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.205023 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default I0417 18:40:30.227247 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-216.ec2.internal in network "default" I0417 18:40:30.227254 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "ip-10-0-0-216.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:30.227273 3681 node_tracker.go:235] Node ip-10-0-0-216.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-216.ec2.internal" I0417 18:40:30.227289 3681 node_tracker.go:169] Node ip-10-0-0-216.ec2.internal switch + router changed, syncing services in network "default" I0417 18:40:30.227335 3681 services_controller.go:574] Full service sync requested for network=default W0417 18:40:30.227425 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0417 18:40:30.227526 3681 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.227542 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.227548 3681 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.227577 3681 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0417 18:40:30.227870 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0417 18:40:30.227899 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.227909 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.227918 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.227956 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.227966 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default I0417 18:40:30.253965 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "ip-10-0-0-216.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false W0417 18:40:30.307198 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.307226 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.308086 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.308107 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.308116 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default I0417 18:40:30.402082 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-216.ec2.internal in network "default" I0417 18:40:30.402171 3681 default_network_controller.go:1025] Node "ip-10-0-0-216.ec2.internal" in remote zone "ip-10-0-0-216.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0417 18:40:30.402226 3681 node_tracker.go:169] Node ip-10-0-0-216.ec2.internal switch + router changed, syncing services in network "default" I0417 18:40:30.402239 3681 services_controller.go:574] Full service sync requested for network=default W0417 18:40:30.402670 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0417 18:40:30.402714 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.402729 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.402742 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.402776 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.402791 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.402885 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0417 18:40:30.403033 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.403049 3681 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.403056 3681 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.403127 3681 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0417 18:40:30.403460 3681 zone_ic_handler.go:195] Time taken to create transit switch: 364.149µs I0417 18:40:30.403478 3681 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-216.ec2.internal for the network default I0417 18:40:30.404950 3681 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-216.ec2.internal" on network "default" took: 1.870483ms I0417 18:40:30.622188 3681 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-68.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0417 18:40:30.622212 3681 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-68.ec2.internal", IP: 10.0.1.68 to PMTUD blocking rules I0417 18:40:30.622230 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-68.ec2.internal in network "default" I0417 18:40:30.622242 3681 egressfirewall.go:723] Syncing node "ip-10-0-1-68.ec2.internal" for egress firewall I0417 18:40:30.622199 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-68.ec2.internal I0417 18:40:30.622280 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-68.ec2.internal: 84.512µs I0417 18:40:30.622284 3681 node_tracker.go:169] Node ip-10-0-1-68.ec2.internal switch + router changed, syncing services in network "default" I0417 18:40:30.622302 3681 services_controller.go:574] Full service sync requested for network=default W0417 18:40:30.622523 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0417 18:40:30.622590 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.622603 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.622608 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.622618 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.622625 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.622704 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0417 18:40:30.622764 3681 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.622780 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.622789 3681 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.622836 3681 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0417 18:40:30.628408 3681 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-68.ec2.internal. Updating no re-route policies I0417 18:40:30.636646 3681 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-68.ec2.internal", IP: 10.0.1.68 to PMTUD blocking rules I0417 18:40:30.636662 3681 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-68.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0417 18:40:30.958085 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-216.ec2.internal I0417 18:40:30.958109 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-216.ec2.internal: 28.918µs I0417 18:40:30.958114 3681 egressfirewall.go:723] Syncing node "ip-10-0-0-216.ec2.internal" for egress firewall I0417 18:40:30.958082 3681 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-216.ec2.internal in network "default" I0417 18:40:30.958180 3681 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-216.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0417 18:40:30.958201 3681 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-216.ec2.internal", IP: 10.0.0.216 to PMTUD blocking rules I0417 18:40:30.958185 3681 node_tracker.go:169] Node ip-10-0-0-216.ec2.internal switch + router changed, syncing services in network "default" I0417 18:40:30.958277 3681 services_controller.go:574] Full service sync requested for network=default W0417 18:40:30.958412 3681 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0417 18:40:30.958608 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0417 18:40:30.958621 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.958631 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.958635 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:30.958800 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.958814 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:30.958826 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0417 18:40:30.958896 3681 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.958905 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:30.958909 3681 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default I0417 18:40:30.964432 3681 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-216.ec2.internal. Updating no re-route policies I0417 18:40:30.972384 3681 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-216.ec2.internal", IP: 10.0.0.216 to PMTUD blocking rules I0417 18:40:30.972401 3681 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-216.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0417 18:40:31.068546 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0417 18:40:31.068571 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0417 18:40:31.138631 3681 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0417 18:40:31.151479 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:31.151499 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0417 18:40:31.151506 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default I0417 18:40:31.259637 3681 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0417 18:40:32.747029 3681 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5vngs] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-5vngs for pod on switch ip-10-0-2-177.ec2.internal I0417 18:40:32.747174 3681 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5vngs I0417 18:40:32.761084 3681 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5vngs] pod update took 13.936596ms I0417 18:40:32.761112 3681 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5vngs] addLogicalPort annotation time took 13.970631ms I0417 18:40:32.762466 3681 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5vngs] addLogicalPort took 15.45334ms, libovsdb time 955.18µs W0417 18:40:32.890963 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0417 18:40:33.156936 3681 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5vngs 81e7843141f1563fd014165aa229e151c161c74709896d16a0a02f90b97cbf62 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5vngs 81e7843141f1563fd014165aa229e151c161c74709896d16a0a02f90b97cbf62 network default NAD default NAD key ] I0417 18:40:33.167124 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-5vngs, hostIfaceName: 81e7843141f1563, network: default, NAD default, SandboxID: "81e7843141f1563fd014165aa229e151c161c74709896d16a0a02f90b97cbf62", PCI device ID: , UID: "3ecff34f-d968-4836-a363-0a511194ed3b", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0417 18:40:33.218266 3681 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5vngs 81e7843141f1563fd014165aa229e151c161c74709896d16a0a02f90b97cbf62 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-5vngs 81e7843141f1563fd014165aa229e151c161c74709896d16a0a02f90b97cbf62 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:d3:82:5c:65:d3\",\"name\":\"81e7843141f1563\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a86dc409-081e-411a-a86a-5d4997df7e6f\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:40:34.456023 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-68.ec2.internal I0417 18:40:34.456060 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-68.ec2.internal: 47.77µs W0417 18:40:34.528474 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:34.528578 3681 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:34.528594 3681 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default I0417 18:40:35.008082 3681 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-216.ec2.internal I0417 18:40:35.008117 3681 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-216.ec2.internal: 40.229µs W0417 18:40:35.079662 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0417 18:40:35.115433 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0417 18:40:35.264850 3681 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0417 18:40:35.793982 3681 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0417 18:40:37.020107 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:37.020133 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:37.020141 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0417 18:40:45.009008 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0417 18:40:52.770541 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics I0417 18:41:01.128671 3681 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0417 18:41:09.917325 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0417 18:41:10.875429 3681 base_network_controller_pods.go:487] [default/openshift-console/console-774cbc7cbd-8j2zj] creating logical port openshift-console_console-774cbc7cbd-8j2zj for pod on switch ip-10-0-2-177.ec2.internal I0417 18:41:10.875543 3681 kube.go:256] Updating pod openshift-console/console-774cbc7cbd-8j2zj I0417 18:41:10.893964 3681 pod.go:62] [openshift-console/console-774cbc7cbd-8j2zj] pod update took 18.451135ms I0417 18:41:10.893995 3681 base_network_controller_pods.go:951] [default/openshift-console/console-774cbc7cbd-8j2zj] addLogicalPort annotation time took 18.487585ms I0417 18:41:10.895389 3681 pods.go:271] [openshift-console/console-774cbc7cbd-8j2zj] addLogicalPort took 19.974142ms, libovsdb time 973.495µs I0417 18:41:11.237949 3681 cni.go:452] [openshift-console/console-774cbc7cbd-8j2zj 898b27a637378c52d308722d249191ef5a33ed5b9b51e38f5b8fad431a91e47d network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-774cbc7cbd-8j2zj 898b27a637378c52d308722d249191ef5a33ed5b9b51e38f5b8fad431a91e47d network default NAD default NAD key ] I0417 18:41:11.247589 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-774cbc7cbd-8j2zj, hostIfaceName: 898b27a637378c5, network: default, NAD default, SandboxID: "898b27a637378c52d308722d249191ef5a33ed5b9b51e38f5b8fad431a91e47d", PCI device ID: , UID: "2bab432a-52f9-4b07-a69e-6586bf2775a0", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0417 18:41:11.301507 3681 cni.go:473] [openshift-console/console-774cbc7cbd-8j2zj 898b27a637378c52d308722d249191ef5a33ed5b9b51e38f5b8fad431a91e47d network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-774cbc7cbd-8j2zj 898b27a637378c52d308722d249191ef5a33ed5b9b51e38f5b8fad431a91e47d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:d3:46:3b:f9:8d\",\"name\":\"898b27a637378c5\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f394a69a-b079-4717-bca3-8883dbb870d0\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:41:14.237079 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0417 18:41:15.485667 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0417 18:41:15.514138 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0417 18:41:15.565818 3681 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0417 18:41:15.565838 3681 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0417 18:41:15.565843 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0417 18:41:15.612971 3681 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0417 18:41:15.612989 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0417 18:41:15.973019 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0417 18:41:16.002428 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0417 18:41:16.046997 3681 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0417 18:41:16.047017 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0417 18:41:16.138452 3681 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0417 18:41:16.138483 3681 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0417 18:41:16.138490 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0417 18:41:16.974877 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/monitoring-plugin W0417 18:41:17.004626 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0417 18:41:17.004644 3681 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0417 18:41:17.004649 3681 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0417 18:41:17.004653 3681 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0417 18:41:17.459544 3681 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0417 18:41:17.459575 3681 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0417 18:41:17.459586 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0417 18:41:17.459594 3681 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0417 18:41:19.942962 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0417 18:41:19.952787 3681 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-774cbc7cbd-blm48 IP address from the namespace address-set, err: pod openshift-console/console-774cbc7cbd-blm48: no pod IPs found W0417 18:41:20.263950 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0417 18:41:20.263980 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0417 18:41:20.264032 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0417 18:41:21.266338 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0417 18:41:21.297777 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0417 18:41:21.481021 3681 cni.go:452] [openshift-image-registry/image-registry-759bcd5579-sn6lt 519395bc8cd67261dd0d93346924cc62372f584b467a23228bfecd9449ff3073 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-759bcd5579-sn6lt 519395bc8cd67261dd0d93346924cc62372f584b467a23228bfecd9449ff3073 network default NAD default NAD key ] I0417 18:41:21.563118 3681 cni.go:473] [openshift-image-registry/image-registry-759bcd5579-sn6lt 519395bc8cd67261dd0d93346924cc62372f584b467a23228bfecd9449ff3073 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-759bcd5579-sn6lt 519395bc8cd67261dd0d93346924cc62372f584b467a23228bfecd9449ff3073 network default NAD default NAD key default], result "{}", err I0417 18:41:21.616423 3681 obj_retry.go:498] Detected object openshift-image-registry/image-registry-759bcd5579-sn6lt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:41:21.616462 3681 pods.go:176] Deleting pod: openshift-image-registry/image-registry-759bcd5579-sn6lt I0417 18:41:21.618160 3681 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-759bcd5579-sn6lt, ips: 10.128.0.5 I0417 18:41:21.618190 3681 obj_retry.go:498] Detected object openshift-image-registry/image-registry-759bcd5579-sn6lt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:41:22.268510 3681 obj_retry.go:492] Detected object openshift-image-registry/image-registry-759bcd5579-sn6lt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:41:22.268533 3681 obj_retry.go:492] Detected object openshift-image-registry/image-registry-759bcd5579-sn6lt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:41:22.276267 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-759bcd5579-sn6lt I0417 18:41:22.276300 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-759bcd5579-sn6lt W0417 18:41:30.308736 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0417 18:41:30.308773 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0417 18:41:30.308827 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0417 18:41:31.317811 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0417 18:41:35.333466 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0417 18:41:35.333490 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0417 18:41:35.333534 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0417 18:41:36.337884 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0417 18:41:41.253497 3681 obj_retry.go:498] Detected object openshift-console/console-78b9644658-c49xn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:41:41.254064 3681 obj_retry.go:498] Detected object openshift-console/console-78b9644658-c49xn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:41:42.088930 3681 obj_retry.go:492] Detected object openshift-console/console-78b9644658-c49xn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:41:42.088959 3681 obj_retry.go:492] Detected object openshift-console/console-78b9644658-c49xn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:41:42.094266 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-78b9644658-c49xn I0417 18:41:42.094291 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-78b9644658-c49xn I0417 18:41:45.281136 3681 obj_retry.go:498] Detected object openshift-console/console-78b9644658-d5472 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:41:45.281685 3681 obj_retry.go:498] Detected object openshift-console/console-78b9644658-d5472 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:41:45.571051 3681 obj_retry.go:492] Detected object openshift-console/console-78b9644658-d5472 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:41:45.571081 3681 obj_retry.go:492] Detected object openshift-console/console-78b9644658-d5472 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:41:45.576620 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-78b9644658-d5472 I0417 18:41:45.576648 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-78b9644658-d5472 I0417 18:41:47.193389 3681 cni.go:452] [openshift-ingress/router-default-759fbb4fcf-xt7hr d3420907029eda2f108ba93ed7f69741df979fce0c22db961ec8aa28ad4e85de network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-759fbb4fcf-xt7hr d3420907029eda2f108ba93ed7f69741df979fce0c22db961ec8aa28ad4e85de network default NAD default NAD key ] I0417 18:41:47.282806 3681 cni.go:473] [openshift-ingress/router-default-759fbb4fcf-xt7hr d3420907029eda2f108ba93ed7f69741df979fce0c22db961ec8aa28ad4e85de network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-759fbb4fcf-xt7hr d3420907029eda2f108ba93ed7f69741df979fce0c22db961ec8aa28ad4e85de network default NAD default NAD key default], result "{}", err I0417 18:41:47.330839 3681 obj_retry.go:498] Detected object openshift-ingress/router-default-759fbb4fcf-xt7hr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:41:47.330896 3681 pods.go:176] Deleting pod: openshift-ingress/router-default-759fbb4fcf-xt7hr I0417 18:41:47.332327 3681 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-759fbb4fcf-xt7hr, ips: 10.128.0.16 I0417 18:41:47.332358 3681 obj_retry.go:498] Detected object openshift-ingress/router-default-759fbb4fcf-xt7hr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:41:47.684976 3681 obj_retry.go:492] Detected object openshift-ingress/router-default-759fbb4fcf-xt7hr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:41:47.685002 3681 obj_retry.go:492] Detected object openshift-ingress/router-default-759fbb4fcf-xt7hr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:41:47.692487 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-759fbb4fcf-xt7hr I0417 18:41:47.692516 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-759fbb4fcf-xt7hr I0417 18:42:17.340418 3681 cni.go:452] [openshift-console/console-774cbc7cbd-8j2zj 898b27a637378c52d308722d249191ef5a33ed5b9b51e38f5b8fad431a91e47d network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-774cbc7cbd-8j2zj 898b27a637378c52d308722d249191ef5a33ed5b9b51e38f5b8fad431a91e47d network default NAD default NAD key ] I0417 18:42:17.425571 3681 cni.go:473] [openshift-console/console-774cbc7cbd-8j2zj 898b27a637378c52d308722d249191ef5a33ed5b9b51e38f5b8fad431a91e47d network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-774cbc7cbd-8j2zj 898b27a637378c52d308722d249191ef5a33ed5b9b51e38f5b8fad431a91e47d network default NAD default NAD key default], result "{}", err I0417 18:42:17.473839 3681 obj_retry.go:498] Detected object openshift-console/console-774cbc7cbd-8j2zj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:17.473879 3681 pods.go:176] Deleting pod: openshift-console/console-774cbc7cbd-8j2zj I0417 18:42:17.475340 3681 pods.go:236] Attempting to release IPs for pod: openshift-console/console-774cbc7cbd-8j2zj, ips: 10.128.0.26 I0417 18:42:17.475373 3681 obj_retry.go:498] Detected object openshift-console/console-774cbc7cbd-8j2zj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:17.513477 3681 obj_retry.go:492] Detected object openshift-console/console-774cbc7cbd-8j2zj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:17.513504 3681 obj_retry.go:492] Detected object openshift-console/console-774cbc7cbd-8j2zj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:17.522329 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-774cbc7cbd-8j2zj I0417 18:42:17.522360 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-774cbc7cbd-8j2zj I0417 18:42:25.409574 3681 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653905j272 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:25.410212 3681 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653905j272 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:25.410240 3681 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653905j272 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:42:25.410252 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0417 18:42:25.410275 3681 obj_retry.go:498] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653905j272 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:42:25.410283 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0417 18:42:26.436241 3681 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653905j272 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:26.436270 3681 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653905j272 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:26.436288 3681 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653905j272 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:26.436301 3681 obj_retry.go:492] Detected object openshift-deployment-validation-operator/ca5b21c3869491e79768d2cfaf75f947507507b9e3b372dfa4bf5653905j272 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:27.376893 3681 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:27.377499 3681 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:27.741506 3681 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:27.741535 3681 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:27.749171 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0417 18:42:27.749196 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0417 18:42:28.236338 3681 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0417 18:42:28.246294 3681 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0417 18:42:30.770217 3681 base_network_controller_pods.go:487] [default/openshift-console/console-69c69cdb5c-qs8lt] creating logical port openshift-console_console-69c69cdb5c-qs8lt for pod on switch ip-10-0-2-177.ec2.internal I0417 18:42:30.770332 3681 kube.go:256] Updating pod openshift-console/console-69c69cdb5c-qs8lt I0417 18:42:30.785141 3681 pod.go:62] [openshift-console/console-69c69cdb5c-qs8lt] pod update took 14.842731ms I0417 18:42:30.785166 3681 base_network_controller_pods.go:951] [default/openshift-console/console-69c69cdb5c-qs8lt] addLogicalPort annotation time took 14.873171ms I0417 18:42:30.786506 3681 pods.go:271] [openshift-console/console-69c69cdb5c-qs8lt] addLogicalPort took 16.305272ms, libovsdb time 979.811µs I0417 18:42:31.142636 3681 cni.go:452] [openshift-console/console-69c69cdb5c-qs8lt 4a654e3915aa97e2b941f240b26870a06d14554efe82c6cea89b4965128233d0 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-69c69cdb5c-qs8lt 4a654e3915aa97e2b941f240b26870a06d14554efe82c6cea89b4965128233d0 network default NAD default NAD key ] I0417 18:42:31.152197 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-69c69cdb5c-qs8lt, hostIfaceName: 4a654e3915aa97e, network: default, NAD default, SandboxID: "4a654e3915aa97e2b941f240b26870a06d14554efe82c6cea89b4965128233d0", PCI device ID: , UID: "f28432fd-546e-480a-b11e-2b969addaeec", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0417 18:42:31.205549 3681 cni.go:473] [openshift-console/console-69c69cdb5c-qs8lt 4a654e3915aa97e2b941f240b26870a06d14554efe82c6cea89b4965128233d0 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-69c69cdb5c-qs8lt 4a654e3915aa97e2b941f240b26870a06d14554efe82c6cea89b4965128233d0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:d9:a0:59:cc:74\",\"name\":\"4a654e3915aa97e\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85ce5408-a8bd-44ce-a0f9-f69cdf2e4e36\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:42:31.844155 3681 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:31.844769 3681 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:32.776331 3681 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:32.776358 3681 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:32.783280 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0417 18:42:32.783304 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 W0417 18:42:41.631843 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0417 18:42:49.501647 3681 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:49.502291 3681 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:50.386159 3681 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:50.386190 3681 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:50.394395 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0417 18:42:50.394416 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0417 18:42:56.080541 3681 obj_retry.go:498] Detected object openshift-console/console-5cf9589c6c-brv57 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:56.081149 3681 obj_retry.go:498] Detected object openshift-console/console-5cf9589c6c-brv57 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:42:56.416525 3681 obj_retry.go:492] Detected object openshift-console/console-5cf9589c6c-brv57 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:56.416555 3681 obj_retry.go:492] Detected object openshift-console/console-5cf9589c6c-brv57 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:42:56.419492 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5cf9589c6c-brv57 I0417 18:42:56.419519 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5cf9589c6c-brv57 I0417 18:43:06.945368 3681 obj_retry.go:498] Detected object openshift-console/console-5cf9589c6c-4bts6 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:43:06.945948 3681 obj_retry.go:498] Detected object openshift-console/console-5cf9589c6c-4bts6 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:43:07.917876 3681 obj_retry.go:492] Detected object openshift-console/console-5cf9589c6c-4bts6 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:43:07.917902 3681 obj_retry.go:492] Detected object openshift-console/console-5cf9589c6c-4bts6 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:43:07.923521 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5cf9589c6c-4bts6 I0417 18:43:07.923554 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5cf9589c6c-4bts6 I0417 18:43:09.752905 3681 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g 5701fae4858326300d3d95517c528fa6dea982c8a52c4bb4a2e3eb2226768323 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g 5701fae4858326300d3d95517c528fa6dea982c8a52c4bb4a2e3eb2226768323 network default NAD default NAD key ] I0417 18:43:09.829179 3681 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g 5701fae4858326300d3d95517c528fa6dea982c8a52c4bb4a2e3eb2226768323 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g 5701fae4858326300d3d95517c528fa6dea982c8a52c4bb4a2e3eb2226768323 network default NAD default NAD key default], result "{}", err I0417 18:43:09.872763 3681 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:43:09.872803 3681 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g I0417 18:43:09.874330 3681 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g, ips: 10.128.0.21 I0417 18:43:09.874359 3681 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:43:10.772366 3681 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:43:10.772392 3681 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607520-4b58g of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:43:21.947602 3681 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0417 18:43:35.360609 3681 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:43:35.361239 3681 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:43:35.582641 3681 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:43:35.582669 3681 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:43:35.589057 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0417 18:43:35.589085 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0417 18:44:47.062297 3681 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0417 18:44:49.049982 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0417 18:45:03.056055 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=5 I0417 18:45:18.939139 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=84 I0417 18:45:46.076575 3681 namespace.go:98] [tsf] adding namespace I0417 18:45:46.077658 3681 namespace.go:102] [tsf] adding namespace took 1.053789ms I0417 18:45:46.094950 3681 namespace.go:142] [tsf] updating namespace I0417 18:45:46.104922 3681 namespace.go:142] [tsf] updating namespace I0417 18:45:50.281092 3681 namespace.go:98] [cert-manager-operator] adding namespace I0417 18:45:50.282053 3681 namespace.go:102] [cert-manager-operator] adding namespace took 931.258µs I0417 18:45:50.284553 3681 namespace.go:98] [rhtpa-operator] adding namespace I0417 18:45:50.284617 3681 namespace.go:98] [konflux-ui] adding namespace I0417 18:45:50.284747 3681 namespace.go:98] [tsf-keycloak] adding namespace I0417 18:45:50.285362 3681 namespace.go:102] [rhtpa-operator] adding namespace took 791.337µs I0417 18:45:50.285980 3681 namespace.go:102] [konflux-ui] adding namespace took 1.345397ms I0417 18:45:50.286656 3681 namespace.go:102] [tsf-keycloak] adding namespace took 1.889805ms I0417 18:45:50.294586 3681 namespace.go:98] [tsf-tpa] adding namespace I0417 18:45:50.294630 3681 namespace.go:98] [konflux-operator] adding namespace I0417 18:45:50.294785 3681 namespace.go:98] [tsf-tas] adding namespace I0417 18:45:50.294886 3681 namespace.go:98] [rhbk-operator] adding namespace I0417 18:45:50.295233 3681 namespace.go:102] [tsf-tpa] adding namespace took 630.88µs I0417 18:45:50.295737 3681 namespace.go:102] [konflux-operator] adding namespace took 1.092728ms I0417 18:45:50.296214 3681 namespace.go:102] [tsf-tas] adding namespace took 1.413685ms I0417 18:45:50.296717 3681 namespace.go:102] [rhbk-operator] adding namespace took 1.819064ms I0417 18:45:50.301495 3681 namespace.go:142] [cert-manager-operator] updating namespace I0417 18:45:50.324248 3681 namespace.go:142] [cert-manager-operator] updating namespace I0417 18:45:50.324294 3681 namespace.go:142] [tsf-tpa] updating namespace I0417 18:45:50.327392 3681 namespace.go:142] [rhtpa-operator] updating namespace I0417 18:45:50.337293 3681 namespace.go:142] [tsf-keycloak] updating namespace I0417 18:45:50.337340 3681 namespace.go:142] [konflux-ui] updating namespace I0417 18:45:50.338657 3681 namespace.go:142] [konflux-operator] updating namespace I0417 18:45:50.341016 3681 namespace.go:142] [tsf-tas] updating namespace I0417 18:45:50.350842 3681 namespace.go:142] [rhbk-operator] updating namespace I0417 18:45:50.350947 3681 namespace.go:142] [cert-manager-operator] updating namespace I0417 18:45:50.381227 3681 namespace.go:142] [konflux-ui] updating namespace I0417 18:45:50.387278 3681 namespace.go:142] [konflux-ui] updating namespace I0417 18:45:50.387411 3681 namespace.go:142] [tsf-keycloak] updating namespace I0417 18:45:50.397403 3681 namespace.go:142] [tsf-keycloak] updating namespace I0417 18:45:50.413087 3681 namespace.go:142] [tsf-tpa] updating namespace I0417 18:45:50.436504 3681 namespace.go:142] [tsf-tpa] updating namespace I0417 18:45:50.436556 3681 namespace.go:142] [konflux-operator] updating namespace I0417 18:45:50.449675 3681 namespace.go:142] [tsf-tas] updating namespace I0417 18:45:50.454339 3681 namespace.go:142] [konflux-operator] updating namespace I0417 18:45:50.467041 3681 namespace.go:142] [tsf-tas] updating namespace I0417 18:45:50.470539 3681 namespace.go:142] [rhbk-operator] updating namespace I0417 18:45:50.498422 3681 namespace.go:142] [rhbk-operator] updating namespace I0417 18:45:50.498580 3681 namespace.go:142] [rhtpa-operator] updating namespace I0417 18:45:50.498676 3681 namespace.go:142] [rhtpa-operator] updating namespace I0417 18:45:50.942288 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=141 I0417 18:45:51.447323 3681 namespace.go:142] [tsf-tpa] updating namespace I0417 18:45:51.459279 3681 namespace.go:142] [tsf-keycloak] updating namespace I0417 18:45:51.616073 3681 base_network_controller_pods.go:487] [default/tsf/test-tsf-subscriptions] creating logical port tsf_test-tsf-subscriptions for pod on switch ip-10-0-2-177.ec2.internal I0417 18:45:51.616221 3681 kube.go:256] Updating pod tsf/test-tsf-subscriptions I0417 18:45:51.632190 3681 pod.go:62] [tsf/test-tsf-subscriptions] pod update took 16.006677ms I0417 18:45:51.632220 3681 base_network_controller_pods.go:951] [default/tsf/test-tsf-subscriptions] addLogicalPort annotation time took 16.039792ms I0417 18:45:51.633654 3681 pods.go:271] [tsf/test-tsf-subscriptions] addLogicalPort took 17.605314ms, libovsdb time 1.039476ms I0417 18:45:51.996462 3681 cni.go:452] [tsf/test-tsf-subscriptions 0d38e7189c799de8247c5c53adc1f7a9bb7409bf61611e6828fc9d1228e876f4 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-subscriptions 0d38e7189c799de8247c5c53adc1f7a9bb7409bf61611e6828fc9d1228e876f4 network default NAD default NAD key ] I0417 18:45:52.005886 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-subscriptions, hostIfaceName: 0d38e7189c799de, network: default, NAD default, SandboxID: "0d38e7189c799de8247c5c53adc1f7a9bb7409bf61611e6828fc9d1228e876f4", PCI device ID: , UID: "e68af436-11bf-4a60-a749-998482f122ff", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0417 18:45:52.057410 3681 cni.go:473] [tsf/test-tsf-subscriptions 0d38e7189c799de8247c5c53adc1f7a9bb7409bf61611e6828fc9d1228e876f4 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-subscriptions 0d38e7189c799de8247c5c53adc1f7a9bb7409bf61611e6828fc9d1228e876f4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:09:f8:98:d8:ce\",\"name\":\"0d38e7189c799de\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/db2fbaa0-516c-454b-afce-efdad4bd8739\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:45:57.258197 3681 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tb8jh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:45:57.258891 3681 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tb8jh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:45:57.258927 3681 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tb8jh of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:45:57.258938 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0417 18:45:57.258953 3681 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tb8jh of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:45:57.258960 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0417 18:45:58.283213 3681 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tb8jh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:45:58.283242 3681 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tb8jh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:45:58.283260 3681 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tb8jh of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:45:58.283272 3681 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2tb8jh of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:45:59.274910 3681 obj_retry.go:498] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mg9js of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:45:59.275719 3681 obj_retry.go:498] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mg9js of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:45:59.275748 3681 obj_retry.go:498] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mg9js of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:45:59.275756 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0417 18:45:59.275766 3681 obj_retry.go:498] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mg9js of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:45:59.275770 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0417 18:46:00.306318 3681 obj_retry.go:492] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mg9js of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:00.306354 3681 obj_retry.go:492] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mg9js of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:00.306372 3681 obj_retry.go:492] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mg9js of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:00.306388 3681 obj_retry.go:492] Detected object openshift-marketplace/925ad1f05bf386dc21bdfe2f8249c1fbfd04a404dec7a7fb6362d758e5mg9js of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:02.309100 3681 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ececm4j8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:46:02.309663 3681 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ececm4j8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:46:02.309686 3681 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ececm4j8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:46:02.309714 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0417 18:46:02.309725 3681 obj_retry.go:498] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ececm4j8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:46:02.309729 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0417 18:46:02.332130 3681 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568zmjnk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:46:02.332734 3681 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568zmjnk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:46:02.332760 3681 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568zmjnk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:46:02.332769 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0417 18:46:02.332781 3681 obj_retry.go:498] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568zmjnk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:46:02.332785 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0417 18:46:03.347426 3681 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ececm4j8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:03.347451 3681 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ececm4j8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:03.347468 3681 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ececm4j8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:03.347478 3681 obj_retry.go:492] Detected object openshift-marketplace/53634cc716e3784f8e237b128ae14253d65dd14b0e9dcd086f5ff88ececm4j8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:03.374720 3681 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568zmjnk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:03.374752 3681 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568zmjnk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:03.374778 3681 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568zmjnk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:03.374787 3681 obj_retry.go:492] Detected object openshift-marketplace/f79378b9836da0a180436b794e6453783590867ba696f35668b1ac4568zmjnk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:05.346103 3681 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafwxmxd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:46:05.346750 3681 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafwxmxd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:46:05.346789 3681 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafwxmxd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:46:05.346800 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0417 18:46:05.346814 3681 obj_retry.go:498] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafwxmxd of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:46:05.346819 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0417 18:46:05.371681 3681 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7wkx78 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:46:05.372243 3681 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7wkx78 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:46:05.372281 3681 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7wkx78 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:46:05.372294 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0417 18:46:05.372306 3681 obj_retry.go:498] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7wkx78 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0417 18:46:05.372310 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0417 18:46:06.370524 3681 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafwxmxd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:06.370553 3681 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafwxmxd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:06.370570 3681 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafwxmxd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:06.370584 3681 obj_retry.go:492] Detected object openshift-marketplace/30e1fee4a656579f055e90bd50374338b32290af440ed32e709fbffaafwxmxd of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:06.399472 3681 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7wkx78 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:06.399494 3681 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7wkx78 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:06.399507 3681 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7wkx78 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:06.399518 3681 obj_retry.go:492] Detected object openshift-marketplace/18ab2eaa607a00575555d3d5b97fe3206aa96262fc8b848ad325c51ad7wkx78 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:46:14.033155 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0417 18:46:14.335375 3681 namespace.go:98] [cert-manager] adding namespace I0417 18:46:14.336285 3681 namespace.go:102] [cert-manager] adding namespace took 885.78µs I0417 18:46:14.363988 3681 namespace.go:142] [cert-manager] updating namespace I0417 18:46:14.377293 3681 namespace.go:142] [cert-manager] updating namespace W0417 18:46:22.311481 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0417 18:46:22.311504 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0417 18:46:24.120118 3681 namespace.go:142] [openshift-operators] updating namespace I0417 18:46:24.140553 3681 namespace.go:142] [openshift-operators] updating namespace W0417 18:46:30.453635 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0417 18:46:32.054477 3681 namespace.go:142] [openshift-operators] updating namespace W0417 18:46:33.708738 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0417 18:46:38.008496 3681 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9sgpj] creating logical port openshift-operators_rhtas-operator-controller-manager-6987b7f98c-9sgpj for pod on switch ip-10-0-2-177.ec2.internal I0417 18:46:38.008614 3681 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9sgpj I0417 18:46:38.034383 3681 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9sgpj] pod update took 25.796493ms I0417 18:46:38.034408 3681 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9sgpj] addLogicalPort annotation time took 25.827674ms I0417 18:46:38.035835 3681 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9sgpj] addLogicalPort took 27.362443ms, libovsdb time 957.059µs I0417 18:46:38.386121 3681 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9sgpj 02d35d126d528c5ecdec7b21fc253199b0810b8a1210a50e7a4f2248bba16df8 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9sgpj 02d35d126d528c5ecdec7b21fc253199b0810b8a1210a50e7a4f2248bba16df8 network default NAD default NAD key ] I0417 18:46:38.396447 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-6987b7f98c-9sgpj, hostIfaceName: 02d35d126d528c5, network: default, NAD default, SandboxID: "02d35d126d528c5ecdec7b21fc253199b0810b8a1210a50e7a4f2248bba16df8", PCI device ID: , UID: "19181737-dfd6-41ed-a8f1-77014fd70bd0", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0417 18:46:38.449490 3681 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9sgpj 02d35d126d528c5ecdec7b21fc253199b0810b8a1210a50e7a4f2248bba16df8 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-6987b7f98c-9sgpj 02d35d126d528c5ecdec7b21fc253199b0810b8a1210a50e7a4f2248bba16df8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:75:52:89:b9:81\",\"name\":\"02d35d126d528c5\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b4beca78-606a-4857-bead-9b6a06dae70c\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:46:40.941895 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=88 I0417 18:46:41.158163 3681 namespace.go:98] [trusted-artifact-signer] adding namespace I0417 18:46:41.159116 3681 namespace.go:102] [trusted-artifact-signer] adding namespace took 926.307µs I0417 18:46:41.245195 3681 namespace.go:142] [trusted-artifact-signer] updating namespace I0417 18:46:41.280495 3681 namespace.go:142] [trusted-artifact-signer] updating namespace W0417 18:46:41.774761 3681 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0417 18:46:47.111202 3681 namespace.go:98] [openshift-pipelines] adding namespace I0417 18:46:47.112099 3681 namespace.go:102] [openshift-pipelines] adding namespace took 870.243µs I0417 18:46:47.132382 3681 namespace.go:142] [openshift-pipelines] updating namespace I0417 18:46:47.826989 3681 namespace.go:142] [cert-manager] updating namespace I0417 18:46:48.031028 3681 namespace.go:142] [cert-manager-operator] updating namespace I0417 18:46:48.431638 3681 namespace.go:142] [konflux-operator] updating namespace I0417 18:46:48.634583 3681 namespace.go:142] [konflux-ui] updating namespace I0417 18:46:48.830926 3681 namespace.go:142] [open-cluster-management-2po2q8b3ubd89mbta5n7l49aiaaqqcse] updating namespace I0417 18:46:49.238879 3681 namespace.go:142] [rhbk-operator] updating namespace I0417 18:46:49.435842 3681 namespace.go:142] [rhtpa-operator] updating namespace I0417 18:46:49.635583 3681 namespace.go:142] [trusted-artifact-signer] updating namespace I0417 18:46:49.834956 3681 namespace.go:142] [tsf] updating namespace I0417 18:46:50.038089 3681 namespace.go:142] [tsf-keycloak] updating namespace I0417 18:46:50.239940 3681 namespace.go:142] [tsf-tas] updating namespace I0417 18:46:50.440233 3681 namespace.go:142] [tsf-tpa] updating namespace I0417 18:46:50.563845 3681 namespace.go:142] [cert-manager] updating namespace I0417 18:46:50.571879 3681 namespace.go:142] [cert-manager-operator] updating namespace I0417 18:46:50.582538 3681 namespace.go:142] [default] updating namespace I0417 18:46:50.593089 3681 namespace.go:142] [konflux-operator] updating namespace I0417 18:46:50.601494 3681 namespace.go:142] [konflux-ui] updating namespace I0417 18:46:50.611997 3681 namespace.go:142] [open-cluster-management-2po2q8b3ubd89mbta5n7l49aiaaqqcse] updating namespace I0417 18:46:50.617873 3681 namespace.go:142] [openshift] updating namespace I0417 18:46:50.626997 3681 namespace.go:142] [rhbk-operator] updating namespace I0417 18:46:50.635496 3681 namespace.go:142] [rhtpa-operator] updating namespace I0417 18:46:50.644026 3681 namespace.go:142] [trusted-artifact-signer] updating namespace I0417 18:46:50.658067 3681 namespace.go:142] [tsf] updating namespace I0417 18:46:50.679071 3681 namespace.go:142] [tsf-keycloak] updating namespace I0417 18:46:50.702489 3681 namespace.go:142] [tsf-tas] updating namespace I0417 18:46:50.720235 3681 namespace.go:142] [tsf-tpa] updating namespace I0417 18:46:50.819227 3681 namespace.go:142] [cert-manager] updating namespace I0417 18:46:50.921467 3681 namespace.go:142] [cert-manager-operator] updating namespace I0417 18:46:51.019963 3681 namespace.go:142] [default] updating namespace I0417 18:46:51.121665 3681 namespace.go:142] [konflux-operator] updating namespace I0417 18:46:51.224663 3681 namespace.go:142] [konflux-ui] updating namespace I0417 18:46:51.333120 3681 namespace.go:142] [open-cluster-management-2po2q8b3ubd89mbta5n7l49aiaaqqcse] updating namespace I0417 18:46:51.418672 3681 namespace.go:142] [openshift] updating namespace I0417 18:46:51.521665 3681 namespace.go:142] [rhbk-operator] updating namespace I0417 18:46:51.622829 3681 namespace.go:142] [rhtpa-operator] updating namespace I0417 18:46:51.721083 3681 namespace.go:142] [trusted-artifact-signer] updating namespace I0417 18:46:51.819364 3681 namespace.go:142] [tsf] updating namespace I0417 18:46:51.926071 3681 namespace.go:142] [tsf-keycloak] updating namespace I0417 18:46:52.021854 3681 namespace.go:142] [tsf-tas] updating namespace I0417 18:46:52.121472 3681 namespace.go:142] [tsf-tpa] updating namespace I0417 18:46:52.501503 3681 namespace.go:142] [openshift-pipelines] updating namespace W0417 18:46:52.724544 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0417 18:46:53.038271 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0417 18:47:01.771745 3681 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-dh8nk] creating logical port openshift-pipelines_tekton-pipelines-controller-84d6d58f5c-dh8nk for pod on switch ip-10-0-2-177.ec2.internal I0417 18:47:01.771846 3681 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-dh8nk I0417 18:47:01.787577 3681 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-dh8nk] pod update took 15.756666ms I0417 18:47:01.787603 3681 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-dh8nk] addLogicalPort annotation time took 15.787662ms I0417 18:47:01.788986 3681 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-dh8nk] addLogicalPort took 17.263014ms, libovsdb time 872.202µs I0417 18:47:01.975453 3681 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz for pod on switch ip-10-0-2-177.ec2.internal I0417 18:47:01.975558 3681 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz I0417 18:47:01.989677 3681 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz] pod update took 14.145295ms I0417 18:47:01.989716 3681 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz] addLogicalPort annotation time took 14.190671ms I0417 18:47:01.990966 3681 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz] addLogicalPort took 15.541817ms, libovsdb time 832.586µs I0417 18:47:02.135314 3681 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-dh8nk 9f1b30d782efd18daf5ecf92b36ab78ac0b814c48b2b2d3a8aec1b671bc7c497 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-dh8nk 9f1b30d782efd18daf5ecf92b36ab78ac0b814c48b2b2d3a8aec1b671bc7c497 network default NAD default NAD key ] I0417 18:47:02.146503 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-84d6d58f5c-dh8nk, hostIfaceName: 9f1b30d782efd18, network: default, NAD default, SandboxID: "9f1b30d782efd18daf5ecf92b36ab78ac0b814c48b2b2d3a8aec1b671bc7c497", PCI device ID: , UID: "f73e0d0c-d5a8-4140-869d-83ccb050a29a", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0417 18:47:02.201335 3681 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-dh8nk 9f1b30d782efd18daf5ecf92b36ab78ac0b814c48b2b2d3a8aec1b671bc7c497 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-84d6d58f5c-dh8nk 9f1b30d782efd18daf5ecf92b36ab78ac0b814c48b2b2d3a8aec1b671bc7c497 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"36:03:88:e6:ea:93\",\"name\":\"9f1b30d782efd18\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/db40cae5-8abc-4a9b-8ecb-99c40b1bab7d\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:47:02.342659 3681 cni.go:452] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz d444c36cbb0836f228106a94a56fcebc7b7cce839e970679a4f4225dcc6493ef network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz d444c36cbb0836f228106a94a56fcebc7b7cce839e970679a4f4225dcc6493ef network default NAD default NAD key ] I0417 18:47:02.353509 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz, hostIfaceName: d444c36cbb0836f, network: default, NAD default, SandboxID: "d444c36cbb0836f228106a94a56fcebc7b7cce839e970679a4f4225dcc6493ef", PCI device ID: , UID: "d800f498-f752-41f0-9cad-0a15ddc696e8", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0417 18:47:02.409254 3681 cni.go:473] [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz d444c36cbb0836f228106a94a56fcebc7b7cce839e970679a4f4225dcc6493ef network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-56879bb9d7-pb2hz d444c36cbb0836f228106a94a56fcebc7b7cce839e970679a4f4225dcc6493ef network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:e8:73:65:24:bb\",\"name\":\"d444c36cbb0836f\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/36caecf8-28f0-4446-a87b-7b158a1a87c9\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:47:03.815322 3681 cni.go:452] [tsf/test-tsf-subscriptions 0d38e7189c799de8247c5c53adc1f7a9bb7409bf61611e6828fc9d1228e876f4 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-subscriptions 0d38e7189c799de8247c5c53adc1f7a9bb7409bf61611e6828fc9d1228e876f4 network default NAD default NAD key ] I0417 18:47:03.907897 3681 cni.go:473] [tsf/test-tsf-subscriptions 0d38e7189c799de8247c5c53adc1f7a9bb7409bf61611e6828fc9d1228e876f4 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-subscriptions 0d38e7189c799de8247c5c53adc1f7a9bb7409bf61611e6828fc9d1228e876f4 network default NAD default NAD key default], result "{}", err I0417 18:47:03.977737 3681 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:47:03.977789 3681 pods.go:176] Deleting pod: tsf/test-tsf-subscriptions I0417 18:47:03.979618 3681 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-subscriptions, ips: 10.128.0.28 I0417 18:47:03.979650 3681 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:47:03.999554 3681 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:47:03.999597 3681 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:47:04.006397 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0417 18:47:04.006497 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0417 18:47:05.275230 3681 base_network_controller_pods.go:487] [default/tsf/test-tsf-infrastructure] creating logical port tsf_test-tsf-infrastructure for pod on switch ip-10-0-2-177.ec2.internal I0417 18:47:05.275467 3681 kube.go:256] Updating pod tsf/test-tsf-infrastructure I0417 18:47:05.289989 3681 pod.go:62] [tsf/test-tsf-infrastructure] pod update took 14.559295ms I0417 18:47:05.290084 3681 base_network_controller_pods.go:951] [default/tsf/test-tsf-infrastructure] addLogicalPort annotation time took 14.659081ms I0417 18:47:05.291419 3681 pods.go:271] [tsf/test-tsf-infrastructure] addLogicalPort took 16.215148ms, libovsdb time 980.608µs W0417 18:47:05.561113 3681 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0417 18:47:05.561138 3681 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0417 18:47:05.561144 3681 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller I0417 18:47:05.728134 3681 cni.go:452] [tsf/test-tsf-infrastructure 8f8b0b103f687ef868a1196724ca1388ec614f6d23128bd7ef6ff52f9f04de21 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-infrastructure 8f8b0b103f687ef868a1196724ca1388ec614f6d23128bd7ef6ff52f9f04de21 network default NAD default NAD key ] I0417 18:47:05.737812 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-infrastructure, hostIfaceName: 8f8b0b103f687ef, network: default, NAD default, SandboxID: "8f8b0b103f687ef868a1196724ca1388ec614f6d23128bd7ef6ff52f9f04de21", PCI device ID: , UID: "75d6d96d-e1ba-450a-904a-ec38ccf02c68", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0417 18:47:05.796130 3681 cni.go:473] [tsf/test-tsf-infrastructure 8f8b0b103f687ef868a1196724ca1388ec614f6d23128bd7ef6ff52f9f04de21 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-infrastructure 8f8b0b103f687ef868a1196724ca1388ec614f6d23128bd7ef6ff52f9f04de21 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:da:ac:66:86:ae\",\"name\":\"8f8b0b103f687ef\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/21888e2d-72cb-4493-8cc4-2123406dffcd\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:47:05.855422 3681 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0417 18:47:05.855447 3681 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0417 18:47:05.855454 3681 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0417 18:47:05.855460 3681 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0417 18:47:07.838094 3681 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0417 18:47:07.838118 3681 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0417 18:47:07.838124 3681 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0417 18:47:08.155476 3681 base_network_controller_pods.go:487] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-2xbrp] creating logical port tsf-tpa_tpa-pgsql-bee-5df6ff8bdf-2xbrp for pod on switch ip-10-0-2-177.ec2.internal I0417 18:47:08.155590 3681 kube.go:256] Updating pod tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-2xbrp I0417 18:47:08.159194 3681 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-r552c] creating logical port tsf-keycloak_keycloak-pgsql-bee-744f6fc4d9-r552c for pod on switch ip-10-0-2-177.ec2.internal I0417 18:47:08.159362 3681 kube.go:256] Updating pod tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-r552c I0417 18:47:08.170825 3681 pod.go:62] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-2xbrp] pod update took 15.261407ms I0417 18:47:08.170857 3681 base_network_controller_pods.go:951] [default/tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-2xbrp] addLogicalPort annotation time took 15.297723ms I0417 18:47:08.171600 3681 pod.go:62] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-r552c] pod update took 12.270437ms I0417 18:47:08.171677 3681 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-r552c] addLogicalPort annotation time took 12.34801ms I0417 18:47:08.172744 3681 pods.go:271] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-2xbrp] addLogicalPort took 17.289289ms, libovsdb time 1.343847ms I0417 18:47:08.173014 3681 pods.go:271] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-r552c] addLogicalPort took 13.836503ms, libovsdb time 882.27µs I0417 18:47:10.939657 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0417 18:47:12.123652 3681 cni.go:452] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-2xbrp 4bee6a7e69e0bc195399175ef67ed36bd2fcf286cfc834c83cf108e43c67ecc9 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-2xbrp 4bee6a7e69e0bc195399175ef67ed36bd2fcf286cfc834c83cf108e43c67ecc9 network default NAD default NAD key ] I0417 18:47:12.133685 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: tpa-pgsql-bee-5df6ff8bdf-2xbrp, hostIfaceName: 4bee6a7e69e0bc1, network: default, NAD default, SandboxID: "4bee6a7e69e0bc195399175ef67ed36bd2fcf286cfc834c83cf108e43c67ecc9", PCI device ID: , UID: "5b6b2b1a-9eae-4f16-92c0-a5f2ec60efe4", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0417 18:47:12.188294 3681 cni.go:473] [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-2xbrp 4bee6a7e69e0bc195399175ef67ed36bd2fcf286cfc834c83cf108e43c67ecc9 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/tpa-pgsql-bee-5df6ff8bdf-2xbrp 4bee6a7e69e0bc195399175ef67ed36bd2fcf286cfc834c83cf108e43c67ecc9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:41:37:6b:a7:c4\",\"name\":\"4bee6a7e69e0bc1\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/60aed6e3-a0bf-4af5-a168-9dfe47cdfb85\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:47:12.861018 3681 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0417 18:47:14.237182 3681 cni.go:452] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-r552c 20d6f24d51b96e49e6457e454f5f0904872a05ac370da3a4906bf6fa80d88ce8 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-r552c 20d6f24d51b96e49e6457e454f5f0904872a05ac370da3a4906bf6fa80d88ce8 network default NAD default NAD key ] I0417 18:47:14.247440 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-pgsql-bee-744f6fc4d9-r552c, hostIfaceName: 20d6f24d51b96e4, network: default, NAD default, SandboxID: "20d6f24d51b96e49e6457e454f5f0904872a05ac370da3a4906bf6fa80d88ce8", PCI device ID: , UID: "4fe50e1b-c24e-4ed7-b9aa-77f9c02790d6", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0417 18:47:14.303872 3681 cni.go:473] [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-r552c 20d6f24d51b96e49e6457e454f5f0904872a05ac370da3a4906bf6fa80d88ce8 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-pgsql-bee-744f6fc4d9-r552c 20d6f24d51b96e49e6457e454f5f0904872a05ac370da3a4906bf6fa80d88ce8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:9c:a2:4f:8d:b3\",\"name\":\"20d6f24d51b96e4\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/61c894c2-cd9d-4dcc-b130-89d5638e907d\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:47:14.873033 3681 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0417 18:47:18.846960 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0417 18:47:18.846983 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0417 18:47:18.846989 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0417 18:47:24.945993 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=104 W0417 18:47:28.954084 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0417 18:47:29.542619 3681 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-wv49q] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-d466f869f-wv49q for pod on switch ip-10-0-2-177.ec2.internal I0417 18:47:29.542754 3681 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-wv49q I0417 18:47:29.562017 3681 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-wv49q] pod update took 19.291973ms I0417 18:47:29.562049 3681 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-wv49q] addLogicalPort annotation time took 19.330471ms I0417 18:47:29.563366 3681 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-wv49q] addLogicalPort took 20.766039ms, libovsdb time 911.413µs I0417 18:47:29.907011 3681 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-wv49q cc10e0c27c2f16e8f3cd5a12dea127d753d1c11070277e8d8c4d304952e6104f network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-wv49q cc10e0c27c2f16e8f3cd5a12dea127d753d1c11070277e8d8c4d304952e6104f network default NAD default NAD key ] I0417 18:47:29.917259 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-d466f869f-wv49q, hostIfaceName: cc10e0c27c2f16e, network: default, NAD default, SandboxID: "cc10e0c27c2f16e8f3cd5a12dea127d753d1c11070277e8d8c4d304952e6104f", PCI device ID: , UID: "db45a194-05c7-4e0a-be0e-c5e52920dcbb", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0417 18:47:29.971160 3681 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-wv49q cc10e0c27c2f16e8f3cd5a12dea127d753d1c11070277e8d8c4d304952e6104f network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-d466f869f-wv49q cc10e0c27c2f16e8f3cd5a12dea127d753d1c11070277e8d8c4d304952e6104f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:02:88:bb:0b:b2\",\"name\":\"cc10e0c27c2f16e\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e984cf77-56bc-4137-8ac5-2cd160e32be7\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:47:32.966601 3681 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0417 18:47:34.881007 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 5432: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0417 18:47:35.885359 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 5432: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0417 18:47:39.002255 3681 cni.go:452] [tsf/test-tsf-infrastructure 8f8b0b103f687ef868a1196724ca1388ec614f6d23128bd7ef6ff52f9f04de21 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-infrastructure 8f8b0b103f687ef868a1196724ca1388ec614f6d23128bd7ef6ff52f9f04de21 network default NAD default NAD key ] I0417 18:47:39.082530 3681 cni.go:473] [tsf/test-tsf-infrastructure 8f8b0b103f687ef868a1196724ca1388ec614f6d23128bd7ef6ff52f9f04de21 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-infrastructure 8f8b0b103f687ef868a1196724ca1388ec614f6d23128bd7ef6ff52f9f04de21 network default NAD default NAD key default], result "{}", err I0417 18:47:39.134181 3681 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:47:39.134216 3681 pods.go:176] Deleting pod: tsf/test-tsf-infrastructure I0417 18:47:39.135542 3681 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-infrastructure, ips: 10.128.0.32 I0417 18:47:39.135578 3681 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:47:39.155116 3681 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:47:39.155140 3681 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:47:39.160363 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0417 18:47:39.160390 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0417 18:47:40.248707 3681 base_network_controller_pods.go:487] [default/tsf/test-tsf-iam] creating logical port tsf_test-tsf-iam for pod on switch ip-10-0-2-177.ec2.internal I0417 18:47:40.248817 3681 kube.go:256] Updating pod tsf/test-tsf-iam I0417 18:47:40.262847 3681 pod.go:62] [tsf/test-tsf-iam] pod update took 14.058749ms I0417 18:47:40.262873 3681 base_network_controller_pods.go:951] [default/tsf/test-tsf-iam] addLogicalPort annotation time took 14.091291ms I0417 18:47:40.264405 3681 pods.go:271] [tsf/test-tsf-iam] addLogicalPort took 15.733554ms, libovsdb time 1.137821ms I0417 18:47:40.612599 3681 cni.go:452] [tsf/test-tsf-iam 8d24db623bc8685f10a2c33768732063e47d7d1c4a0a6298d7cdb19423b56969 network default NAD default NAD key ] ADD starting CNI request [tsf/test-tsf-iam 8d24db623bc8685f10a2c33768732063e47d7d1c4a0a6298d7cdb19423b56969 network default NAD default NAD key ] I0417 18:47:40.622295 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: test-tsf-iam, hostIfaceName: 8d24db623bc8685, network: default, NAD default, SandboxID: "8d24db623bc8685f10a2c33768732063e47d7d1c4a0a6298d7cdb19423b56969", PCI device ID: , UID: "9fb563e4-d3e7-449c-9290-1a6a3b36771a", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0417 18:47:40.677186 3681 cni.go:473] [tsf/test-tsf-iam 8d24db623bc8685f10a2c33768732063e47d7d1c4a0a6298d7cdb19423b56969 network default NAD default NAD key default] ADD finished CNI request [tsf/test-tsf-iam 8d24db623bc8685f10a2c33768732063e47d7d1c4a0a6298d7cdb19423b56969 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:fb:ef:11:7c:39\",\"name\":\"8d24db623bc8685\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0a1c0bfa-f20b-40a3-acb1-7a27a2acc5c5\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:47:41.206642 3681 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0417 18:47:41.207395 3681 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0417 18:47:41.207420 3681 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0417 18:47:41.209834 3681 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0417 18:47:41.571494 3681 base_network_controller_pods.go:487] [default/tsf-keycloak/keycloak-0] creating logical port tsf-keycloak_keycloak-0 for pod on switch ip-10-0-2-177.ec2.internal I0417 18:47:41.571597 3681 kube.go:256] Updating pod tsf-keycloak/keycloak-0 I0417 18:47:41.587345 3681 pod.go:62] [tsf-keycloak/keycloak-0] pod update took 15.774012ms I0417 18:47:41.587376 3681 base_network_controller_pods.go:951] [default/tsf-keycloak/keycloak-0] addLogicalPort annotation time took 15.810001ms I0417 18:47:41.588686 3681 pods.go:271] [tsf-keycloak/keycloak-0] addLogicalPort took 17.213035ms, libovsdb time 887.581µs I0417 18:47:41.936054 3681 cni.go:452] [tsf-keycloak/keycloak-0 a1f3e8e41305ecec30c4615089d72e67037ec98c65e87293906a76f814c2e5fc network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/keycloak-0 a1f3e8e41305ecec30c4615089d72e67037ec98c65e87293906a76f814c2e5fc network default NAD default NAD key ] I0417 18:47:41.945444 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: keycloak-0, hostIfaceName: a1f3e8e41305ece, network: default, NAD default, SandboxID: "a1f3e8e41305ecec30c4615089d72e67037ec98c65e87293906a76f814c2e5fc", PCI device ID: , UID: "98ae1ae9-a961-4526-bd75-aa85f55b52b2", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0417 18:47:42.002092 3681 cni.go:473] [tsf-keycloak/keycloak-0 a1f3e8e41305ecec30c4615089d72e67037ec98c65e87293906a76f814c2e5fc network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/keycloak-0 a1f3e8e41305ecec30c4615089d72e67037ec98c65e87293906a76f814c2e5fc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:57:f3:02:06:2a\",\"name\":\"a1f3e8e41305ece\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e685986c-719b-4f9c-9f4a-2a486fac7487\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:47:43.972386 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8383: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0417 18:47:44.001561 3681 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0417 18:47:44.001684 3681 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-hd82d IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-847f6f7b8-hd82d: no pod IPs found I0417 18:47:47.109161 3681 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0417 18:47:48.058172 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0417 18:47:48.058200 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0417 18:47:48.058211 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0417 18:47:48.058227 3681 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0417 18:47:51.950725 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=958 I0417 18:47:56.067782 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 W0417 18:48:10.932437 3681 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0417 18:48:10.932499 3681 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0417 18:48:10.932508 3681 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0417 18:48:12.173870 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8080: empty IP address endpoints for service tsf-keycloak/keycloak W0417 18:48:12.173892 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak W0417 18:48:12.176731 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 9000: empty IP address endpoints for service tsf-keycloak/keycloak-service W0417 18:48:12.176751 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service tsf-keycloak/keycloak-service I0417 18:48:21.375181 3681 base_network_controller_pods.go:487] [default/tsf-keycloak/tsf-iam-k45zn] creating logical port tsf-keycloak_tsf-iam-k45zn for pod on switch ip-10-0-2-177.ec2.internal I0417 18:48:21.375289 3681 kube.go:256] Updating pod tsf-keycloak/tsf-iam-k45zn I0417 18:48:21.388994 3681 pod.go:62] [tsf-keycloak/tsf-iam-k45zn] pod update took 13.731906ms I0417 18:48:21.389024 3681 base_network_controller_pods.go:951] [default/tsf-keycloak/tsf-iam-k45zn] addLogicalPort annotation time took 13.76631ms I0417 18:48:21.390351 3681 pods.go:271] [tsf-keycloak/tsf-iam-k45zn] addLogicalPort took 15.194474ms, libovsdb time 907.062µs I0417 18:48:21.747573 3681 cni.go:452] [tsf-keycloak/tsf-iam-k45zn 14d6bf46bfe915892dee0ca785126ba09b8d405c2aba3fd1e48dabf9ad3a52b5 network default NAD default NAD key ] ADD starting CNI request [tsf-keycloak/tsf-iam-k45zn 14d6bf46bfe915892dee0ca785126ba09b8d405c2aba3fd1e48dabf9ad3a52b5 network default NAD default NAD key ] I0417 18:48:21.757270 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-keycloak, podName: tsf-iam-k45zn, hostIfaceName: 14d6bf46bfe9158, network: default, NAD default, SandboxID: "14d6bf46bfe915892dee0ca785126ba09b8d405c2aba3fd1e48dabf9ad3a52b5", PCI device ID: , UID: "2f3005ad-fc1c-4d0c-8fc0-06ebbbd3e954", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0417 18:48:21.814786 3681 cni.go:473] [tsf-keycloak/tsf-iam-k45zn 14d6bf46bfe915892dee0ca785126ba09b8d405c2aba3fd1e48dabf9ad3a52b5 network default NAD default NAD key default] ADD finished CNI request [tsf-keycloak/tsf-iam-k45zn 14d6bf46bfe915892dee0ca785126ba09b8d405c2aba3fd1e48dabf9ad3a52b5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"16:81:4c:7f:e7:13\",\"name\":\"14d6bf46bfe9158\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9045a031-0526-4a51-bbcc-a05d4a1b94ee\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:48:25.393858 3681 namespace.go:142] [openshift-pipelines] updating namespace I0417 18:48:31.327599 3681 cni.go:452] [tsf/test-tsf-iam 8d24db623bc8685f10a2c33768732063e47d7d1c4a0a6298d7cdb19423b56969 network default NAD default NAD key ] DEL starting CNI request [tsf/test-tsf-iam 8d24db623bc8685f10a2c33768732063e47d7d1c4a0a6298d7cdb19423b56969 network default NAD default NAD key ] I0417 18:48:31.408095 3681 cni.go:473] [tsf/test-tsf-iam 8d24db623bc8685f10a2c33768732063e47d7d1c4a0a6298d7cdb19423b56969 network default NAD default NAD key default] DEL finished CNI request [tsf/test-tsf-iam 8d24db623bc8685f10a2c33768732063e47d7d1c4a0a6298d7cdb19423b56969 network default NAD default NAD key default], result "{}", err I0417 18:48:31.463159 3681 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:48:31.463202 3681 pods.go:176] Deleting pod: tsf/test-tsf-iam I0417 18:48:31.464837 3681 pods.go:236] Attempting to release IPs for pod: tsf/test-tsf-iam, ips: 10.128.0.36 I0417 18:48:31.464878 3681 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:48:31.503519 3681 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:48:31.503556 3681 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:48:31.508775 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0417 18:48:31.508884 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0417 18:48:32.350927 3681 base_network_controller_pods.go:487] [default/tsf-tas/test-tsf-tas] creating logical port tsf-tas_test-tsf-tas for pod on switch ip-10-0-2-177.ec2.internal I0417 18:48:32.351115 3681 kube.go:256] Updating pod tsf-tas/test-tsf-tas I0417 18:48:32.363650 3681 pod.go:62] [tsf-tas/test-tsf-tas] pod update took 12.567513ms I0417 18:48:32.363675 3681 base_network_controller_pods.go:951] [default/tsf-tas/test-tsf-tas] addLogicalPort annotation time took 12.60007ms I0417 18:48:32.365196 3681 pods.go:271] [tsf-tas/test-tsf-tas] addLogicalPort took 14.292774ms, libovsdb time 1.098187ms I0417 18:48:32.722050 3681 cni.go:452] [tsf-tas/test-tsf-tas e7cea46a3971a29c395f8662fb40055ab167823dce8fb69d59cbb5d2d5ea285d network default NAD default NAD key ] ADD starting CNI request [tsf-tas/test-tsf-tas e7cea46a3971a29c395f8662fb40055ab167823dce8fb69d59cbb5d2d5ea285d network default NAD default NAD key ] I0417 18:48:32.733064 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: test-tsf-tas, hostIfaceName: e7cea46a3971a29, network: default, NAD default, SandboxID: "e7cea46a3971a29c395f8662fb40055ab167823dce8fb69d59cbb5d2d5ea285d", PCI device ID: , UID: "37d5b253-d52e-4a90-a25a-3789d9c551f3", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0417 18:48:32.789579 3681 cni.go:473] [tsf-tas/test-tsf-tas e7cea46a3971a29c395f8662fb40055ab167823dce8fb69d59cbb5d2d5ea285d network default NAD default NAD key default] ADD finished CNI request [tsf-tas/test-tsf-tas e7cea46a3971a29c395f8662fb40055ab167823dce8fb69d59cbb5d2d5ea285d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:10:c8:ac:f1:42\",\"name\":\"e7cea46a3971a29\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0db28fc0-eb0a-4d6e-9537-9d67a59e6e28\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:48:34.542176 3681 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-7f79b5d98c-j8snq] creating logical port tsf-tas_fulcio-server-7f79b5d98c-j8snq for pod on switch ip-10-0-2-177.ec2.internal I0417 18:48:34.542337 3681 kube.go:256] Updating pod tsf-tas/fulcio-server-7f79b5d98c-j8snq I0417 18:48:34.564504 3681 pod.go:62] [tsf-tas/fulcio-server-7f79b5d98c-j8snq] pod update took 22.197507ms I0417 18:48:34.564539 3681 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-7f79b5d98c-j8snq] addLogicalPort annotation time took 22.241584ms I0417 18:48:34.566428 3681 pods.go:271] [tsf-tas/fulcio-server-7f79b5d98c-j8snq] addLogicalPort took 24.272941ms, libovsdb time 1.232689ms I0417 18:48:34.916790 3681 cni.go:452] [tsf-tas/fulcio-server-7f79b5d98c-j8snq e1007a5929557b246a359190545cb19bc5dd8a274d1ef82ece6ab496d14cf988 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-7f79b5d98c-j8snq e1007a5929557b246a359190545cb19bc5dd8a274d1ef82ece6ab496d14cf988 network default NAD default NAD key ] I0417 18:48:34.927304 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-7f79b5d98c-j8snq, hostIfaceName: e1007a5929557b2, network: default, NAD default, SandboxID: "e1007a5929557b246a359190545cb19bc5dd8a274d1ef82ece6ab496d14cf988", PCI device ID: , UID: "905da445-a19e-4505-94d6-8fe02665f743", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0417 18:48:34.985343 3681 cni.go:473] [tsf-tas/fulcio-server-7f79b5d98c-j8snq e1007a5929557b246a359190545cb19bc5dd8a274d1ef82ece6ab496d14cf988 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-7f79b5d98c-j8snq e1007a5929557b246a359190545cb19bc5dd8a274d1ef82ece6ab496d14cf988 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:ee:eb:97:35:50\",\"name\":\"e1007a5929557b2\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8cab70fc-534a-4a15-ae18-4a294d0de069\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:48:35.824790 3681 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-b6lbs] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-b6lbs for pod on switch ip-10-0-2-177.ec2.internal I0417 18:48:35.824919 3681 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-b6lbs I0417 18:48:35.838615 3681 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b6lbs] pod update took 13.723878ms I0417 18:48:35.838649 3681 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-b6lbs] addLogicalPort annotation time took 13.767628ms I0417 18:48:35.840594 3681 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b6lbs] addLogicalPort took 15.826939ms, libovsdb time 1.324725ms I0417 18:48:36.434420 3681 cni.go:452] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b6lbs 55490f8449c967bbffdbb025f6f4d85b7b9dab7bd0611c4f7fbe79822520303f network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b6lbs 55490f8449c967bbffdbb025f6f4d85b7b9dab7bd0611c4f7fbe79822520303f network default NAD default NAD key ] I0417 18:48:36.447715 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-b6lbs, hostIfaceName: 55490f8449c967b, network: default, NAD default, SandboxID: "55490f8449c967bbffdbb025f6f4d85b7b9dab7bd0611c4f7fbe79822520303f", PCI device ID: , UID: "f0cb442b-c17f-43b2-a862-2ee16d22cbe7", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0417 18:48:36.514314 3681 cni.go:473] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b6lbs 55490f8449c967bbffdbb025f6f4d85b7b9dab7bd0611c4f7fbe79822520303f network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-b6lbs 55490f8449c967bbffdbb025f6f4d85b7b9dab7bd0611c4f7fbe79822520303f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:2c:44:19:15:29\",\"name\":\"55490f8449c967b\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a4989440-c8dc-4e14-b04d-68a940573efb\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:48:37.364224 3681 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0417 18:48:37.364248 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0417 18:48:37.387996 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0417 18:48:37.388018 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server I0417 18:48:41.394424 3681 cni.go:452] [tsf-keycloak/tsf-iam-k45zn 14d6bf46bfe915892dee0ca785126ba09b8d405c2aba3fd1e48dabf9ad3a52b5 network default NAD default NAD key ] DEL starting CNI request [tsf-keycloak/tsf-iam-k45zn 14d6bf46bfe915892dee0ca785126ba09b8d405c2aba3fd1e48dabf9ad3a52b5 network default NAD default NAD key ] I0417 18:48:41.475884 3681 cni.go:473] [tsf-keycloak/tsf-iam-k45zn 14d6bf46bfe915892dee0ca785126ba09b8d405c2aba3fd1e48dabf9ad3a52b5 network default NAD default NAD key default] DEL finished CNI request [tsf-keycloak/tsf-iam-k45zn 14d6bf46bfe915892dee0ca785126ba09b8d405c2aba3fd1e48dabf9ad3a52b5 network default NAD default NAD key default], result "{}", err I0417 18:48:41.527545 3681 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-k45zn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:48:41.527589 3681 pods.go:176] Deleting pod: tsf-keycloak/tsf-iam-k45zn I0417 18:48:41.529071 3681 pods.go:236] Attempting to release IPs for pod: tsf-keycloak/tsf-iam-k45zn, ips: 10.128.0.38 I0417 18:48:41.529105 3681 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-k45zn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:48:42.441187 3681 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-k45zn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:48:42.441219 3681 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-k45zn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0417 18:48:44.122481 3681 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0417 18:48:49.347201 3681 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0417 18:48:49.347221 3681 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0417 18:48:49.366320 3681 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0417 18:48:49.384789 3681 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0417 18:48:49.424685 3681 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0417 18:48:52.029211 3681 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=11 I0417 18:48:52.507607 3681 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-xgnds-skmhf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:48:52.508217 3681 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-xgnds-skmhf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:48:53.531127 3681 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-xgnds-skmhf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:48:53.531151 3681 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-xgnds-skmhf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:48:56.029075 3681 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0417 18:49:01.457814 3681 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-rpb4t] creating logical port openshift-pipelines_pipelines-console-plugin-79cb8f5447-rpb4t for pod on switch ip-10-0-2-177.ec2.internal I0417 18:49:01.457916 3681 kube.go:256] Updating pod openshift-pipelines/pipelines-console-plugin-79cb8f5447-rpb4t I0417 18:49:01.469240 3681 pod.go:62] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-rpb4t] pod update took 11.346839ms I0417 18:49:01.469271 3681 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-console-plugin-79cb8f5447-rpb4t] addLogicalPort annotation time took 11.384745ms I0417 18:49:01.470759 3681 pods.go:271] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-rpb4t] addLogicalPort took 12.967366ms, libovsdb time 945.975µs I0417 18:49:01.823544 3681 cni.go:452] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-rpb4t ed34f9a21e0e6f7bf91eb3974d3f44e2604e08fe2e89f74fff81b191c9d43fe9 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-rpb4t ed34f9a21e0e6f7bf91eb3974d3f44e2604e08fe2e89f74fff81b191c9d43fe9 network default NAD default NAD key ] I0417 18:49:01.833328 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-console-plugin-79cb8f5447-rpb4t, hostIfaceName: ed34f9a21e0e6f7, network: default, NAD default, SandboxID: "ed34f9a21e0e6f7bf91eb3974d3f44e2604e08fe2e89f74fff81b191c9d43fe9", PCI device ID: , UID: "2d1085f3-4cd3-4e74-813a-d4d03335d240", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0417 18:49:01.890055 3681 cni.go:473] [openshift-pipelines/pipelines-console-plugin-79cb8f5447-rpb4t ed34f9a21e0e6f7bf91eb3974d3f44e2604e08fe2e89f74fff81b191c9d43fe9 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-console-plugin-79cb8f5447-rpb4t ed34f9a21e0e6f7bf91eb3974d3f44e2604e08fe2e89f74fff81b191c9d43fe9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:1c:00:66:1f:3a\",\"name\":\"ed34f9a21e0e6f7\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/190fee26-fbf8-4acc-8add-f29d5e5ae34b\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:49:21.599887 3681 cni.go:452] [tsf-tas/test-tsf-tas e7cea46a3971a29c395f8662fb40055ab167823dce8fb69d59cbb5d2d5ea285d network default NAD default NAD key ] DEL starting CNI request [tsf-tas/test-tsf-tas e7cea46a3971a29c395f8662fb40055ab167823dce8fb69d59cbb5d2d5ea285d network default NAD default NAD key ] I0417 18:49:21.683438 3681 cni.go:473] [tsf-tas/test-tsf-tas e7cea46a3971a29c395f8662fb40055ab167823dce8fb69d59cbb5d2d5ea285d network default NAD default NAD key default] DEL finished CNI request [tsf-tas/test-tsf-tas e7cea46a3971a29c395f8662fb40055ab167823dce8fb69d59cbb5d2d5ea285d network default NAD default NAD key default], result "{}", err I0417 18:49:21.740400 3681 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:21.740438 3681 pods.go:176] Deleting pod: tsf-tas/test-tsf-tas I0417 18:49:21.741929 3681 pods.go:236] Attempting to release IPs for pod: tsf-tas/test-tsf-tas, ips: 10.128.0.39 I0417 18:49:21.741965 3681 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:21.763006 3681 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:21.763033 3681 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:21.768668 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0417 18:49:21.768710 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0417 18:49:27.485101 3681 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-dpqtw-sgr6c of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:27.485666 3681 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-dpqtw-sgr6c of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:28.363932 3681 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-dpqtw-sgr6c of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:28.363956 3681 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-dpqtw-sgr6c of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:29.653332 3681 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-85757dc6d8-7v8f2] creating logical port tsf-tas_ctlog-85757dc6d8-7v8f2 for pod on switch ip-10-0-2-177.ec2.internal I0417 18:49:29.653431 3681 kube.go:256] Updating pod tsf-tas/ctlog-85757dc6d8-7v8f2 I0417 18:49:29.668961 3681 pod.go:62] [tsf-tas/ctlog-85757dc6d8-7v8f2] pod update took 15.549621ms I0417 18:49:29.668997 3681 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-85757dc6d8-7v8f2] addLogicalPort annotation time took 15.590191ms I0417 18:49:29.670614 3681 pods.go:271] [tsf-tas/ctlog-85757dc6d8-7v8f2] addLogicalPort took 17.296864ms, libovsdb time 990.825µs I0417 18:49:30.068795 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 I0417 18:49:30.625358 3681 cni.go:452] [tsf-tas/ctlog-85757dc6d8-7v8f2 b9c7b3b91d20d2e57f66968385ebf99db85bded8f522b3819b8d24bd82a361e2 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-85757dc6d8-7v8f2 b9c7b3b91d20d2e57f66968385ebf99db85bded8f522b3819b8d24bd82a361e2 network default NAD default NAD key ] I0417 18:49:30.635063 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-85757dc6d8-7v8f2, hostIfaceName: b9c7b3b91d20d2e, network: default, NAD default, SandboxID: "b9c7b3b91d20d2e57f66968385ebf99db85bded8f522b3819b8d24bd82a361e2", PCI device ID: , UID: "15f80025-3b55-4d49-9e8d-b82c52bbbac6", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0417 18:49:30.691220 3681 cni.go:473] [tsf-tas/ctlog-85757dc6d8-7v8f2 b9c7b3b91d20d2e57f66968385ebf99db85bded8f522b3819b8d24bd82a361e2 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-85757dc6d8-7v8f2 b9c7b3b91d20d2e57f66968385ebf99db85bded8f522b3819b8d24bd82a361e2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:d8:2f:e6:8e:35\",\"name\":\"b9c7b3b91d20d2e\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2c043019-d258-4caf-adbc-fc46d6ec8f41\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:49:30.718088 3681 obj_retry.go:498] Detected object tsf/patch-tekton-config-drq9q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:30.718708 3681 obj_retry.go:498] Detected object tsf/patch-tekton-config-drq9q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:31.742107 3681 obj_retry.go:492] Detected object tsf/patch-tekton-config-drq9q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:31.742131 3681 obj_retry.go:492] Detected object tsf/patch-tekton-config-drq9q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:32.511533 3681 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-vqz2t-vmf9w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:32.512507 3681 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-vqz2t-vmf9w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0417 18:49:32.655940 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0417 18:49:33.044381 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=11 I0417 18:49:33.400094 3681 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-vqz2t-vmf9w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:33.400123 3681 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-vqz2t-vmf9w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:36.068861 3681 base_network_controller_pods.go:487] [default/tsf-tas/rekor-search-ui-b8fb4869f-9mmxx] creating logical port tsf-tas_rekor-search-ui-b8fb4869f-9mmxx for pod on switch ip-10-0-2-177.ec2.internal I0417 18:49:36.069033 3681 kube.go:256] Updating pod tsf-tas/rekor-search-ui-b8fb4869f-9mmxx I0417 18:49:36.082666 3681 pod.go:62] [tsf-tas/rekor-search-ui-b8fb4869f-9mmxx] pod update took 13.666981ms I0417 18:49:36.082710 3681 base_network_controller_pods.go:951] [default/tsf-tas/rekor-search-ui-b8fb4869f-9mmxx] addLogicalPort annotation time took 13.703066ms I0417 18:49:36.084314 3681 pods.go:271] [tsf-tas/rekor-search-ui-b8fb4869f-9mmxx] addLogicalPort took 15.473235ms, libovsdb time 1.057455ms I0417 18:49:36.432895 3681 cni.go:452] [tsf-tas/rekor-search-ui-b8fb4869f-9mmxx 8c1eec59b2f4c6d6efe0b348e8a3f5aec1fd320dca811c29b7c276cb652c73df network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-search-ui-b8fb4869f-9mmxx 8c1eec59b2f4c6d6efe0b348e8a3f5aec1fd320dca811c29b7c276cb652c73df network default NAD default NAD key ] I0417 18:49:36.442630 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-search-ui-b8fb4869f-9mmxx, hostIfaceName: 8c1eec59b2f4c6d, network: default, NAD default, SandboxID: "8c1eec59b2f4c6d6efe0b348e8a3f5aec1fd320dca811c29b7c276cb652c73df", PCI device ID: , UID: "e2e3c09e-29cb-47c3-98c2-0bf8e8e54fae", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0417 18:49:36.499721 3681 cni.go:473] [tsf-tas/rekor-search-ui-b8fb4869f-9mmxx 8c1eec59b2f4c6d6efe0b348e8a3f5aec1fd320dca811c29b7c276cb652c73df network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-search-ui-b8fb4869f-9mmxx 8c1eec59b2f4c6d6efe0b348e8a3f5aec1fd320dca811c29b7c276cb652c73df network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:c5:4c:81:6c:20\",\"name\":\"8c1eec59b2f4c6d\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ede45513-262a-43a0-8a23-746efa541d0b\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:49:37.547346 3681 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-5q9wx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:37.548079 3681 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-5q9wx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:38.438120 3681 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-5q9wx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:38.438147 3681 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-5q9wx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:38.469798 3681 obj_retry.go:492] Detected object tsf/patch-tekton-config-drq9q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:38.469823 3681 obj_retry.go:492] Detected object tsf/patch-tekton-config-drq9q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:38.473921 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-drq9q I0417 18:49:38.473954 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-drq9q I0417 18:49:39.125083 3681 base_network_controller_pods.go:487] [default/tsf-tpa/pre-install-tsf-tpa] creating logical port tsf-tpa_pre-install-tsf-tpa for pod on switch ip-10-0-2-177.ec2.internal I0417 18:49:39.125207 3681 kube.go:256] Updating pod tsf-tpa/pre-install-tsf-tpa I0417 18:49:39.140635 3681 pod.go:62] [tsf-tpa/pre-install-tsf-tpa] pod update took 15.453567ms I0417 18:49:39.140663 3681 base_network_controller_pods.go:951] [default/tsf-tpa/pre-install-tsf-tpa] addLogicalPort annotation time took 15.484217ms I0417 18:49:39.142117 3681 pods.go:271] [tsf-tpa/pre-install-tsf-tpa] addLogicalPort took 17.052123ms, libovsdb time 969.871µs I0417 18:49:39.489803 3681 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 7cebf7270dcc3afa6e237d256c89731bb2eb05a1499dd2ec9c2036c2ac511fb2 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/pre-install-tsf-tpa 7cebf7270dcc3afa6e237d256c89731bb2eb05a1499dd2ec9c2036c2ac511fb2 network default NAD default NAD key ] I0417 18:49:39.500436 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: pre-install-tsf-tpa, hostIfaceName: 7cebf7270dcc3af, network: default, NAD default, SandboxID: "7cebf7270dcc3afa6e237d256c89731bb2eb05a1499dd2ec9c2036c2ac511fb2", PCI device ID: , UID: "29e2db92-23b3-4785-bc96-0f04794a3862", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0417 18:49:39.557196 3681 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 7cebf7270dcc3afa6e237d256c89731bb2eb05a1499dd2ec9c2036c2ac511fb2 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/pre-install-tsf-tpa 7cebf7270dcc3afa6e237d256c89731bb2eb05a1499dd2ec9c2036c2ac511fb2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:f9:62:23:32:c0\",\"name\":\"7cebf7270dcc3af\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4d143623-485f-4dbd-a79c-99f118e48267\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:49:39.694158 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0417 18:49:40.622243 3681 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0417 18:49:42.722326 3681 cni.go:452] [tsf-tpa/pre-install-tsf-tpa 7cebf7270dcc3afa6e237d256c89731bb2eb05a1499dd2ec9c2036c2ac511fb2 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/pre-install-tsf-tpa 7cebf7270dcc3afa6e237d256c89731bb2eb05a1499dd2ec9c2036c2ac511fb2 network default NAD default NAD key ] I0417 18:49:42.813391 3681 cni.go:473] [tsf-tpa/pre-install-tsf-tpa 7cebf7270dcc3afa6e237d256c89731bb2eb05a1499dd2ec9c2036c2ac511fb2 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/pre-install-tsf-tpa 7cebf7270dcc3afa6e237d256c89731bb2eb05a1499dd2ec9c2036c2ac511fb2 network default NAD default NAD key default], result "{}", err I0417 18:49:42.867321 3681 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:42.867359 3681 pods.go:176] Deleting pod: tsf-tpa/pre-install-tsf-tpa I0417 18:49:42.868775 3681 pods.go:236] Attempting to release IPs for pod: tsf-tpa/pre-install-tsf-tpa, ips: 10.128.0.45 I0417 18:49:42.868802 3681 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:42.886770 3681 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:42.886828 3681 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:42.893124 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0417 18:49:42.893155 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0417 18:49:43.091831 3681 base_network_controller_pods.go:487] [default/tsf-tpa/test-tsf-tpa] creating logical port tsf-tpa_test-tsf-tpa for pod on switch ip-10-0-2-177.ec2.internal I0417 18:49:43.091933 3681 kube.go:256] Updating pod tsf-tpa/test-tsf-tpa I0417 18:49:43.105614 3681 pod.go:62] [tsf-tpa/test-tsf-tpa] pod update took 13.70619ms I0417 18:49:43.105669 3681 base_network_controller_pods.go:951] [default/tsf-tpa/test-tsf-tpa] addLogicalPort annotation time took 13.764651ms I0417 18:49:43.107164 3681 pods.go:271] [tsf-tpa/test-tsf-tpa] addLogicalPort took 15.349411ms, libovsdb time 944.755µs I0417 18:49:43.456812 3681 cni.go:452] [tsf-tpa/test-tsf-tpa aa62b2e320e4739cfb4d29818d1fe5c58280a0d5659fd1b24108bbe43eb53416 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/test-tsf-tpa aa62b2e320e4739cfb4d29818d1fe5c58280a0d5659fd1b24108bbe43eb53416 network default NAD default NAD key ] I0417 18:49:43.466572 3681 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: test-tsf-tpa, hostIfaceName: aa62b2e320e4739, network: default, NAD default, SandboxID: "aa62b2e320e4739cfb4d29818d1fe5c58280a0d5659fd1b24108bbe43eb53416", PCI device ID: , UID: "174e20d0-dcc1-41a8-b7b1-50a978915109", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0417 18:49:43.521540 3681 cni.go:473] [tsf-tpa/test-tsf-tpa aa62b2e320e4739cfb4d29818d1fe5c58280a0d5659fd1b24108bbe43eb53416 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/test-tsf-tpa aa62b2e320e4739cfb4d29818d1fe5c58280a0d5659fd1b24108bbe43eb53416 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:8d:33:0c:05:39\",\"name\":\"aa62b2e320e4739\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8cf5f6e0-04d8-46be-a7ad-fa57f55eef12\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:49:43.667026 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 6962: empty IP address endpoints for service tsf-tas/ctlog W0417 18:49:44.463837 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0417 18:49:48.640389 3681 obj_retry.go:498] Detected object tsf-tpa/create-db-dlckp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:48.641011 3681 obj_retry.go:498] Detected object tsf-tpa/create-db-dlckp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:49.662879 3681 obj_retry.go:492] Detected object tsf-tpa/create-db-dlckp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:49.662905 3681 obj_retry.go:492] Detected object tsf-tpa/create-db-dlckp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0417 18:49:51.756894 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-search-ui I0417 18:49:52.660737 3681 obj_retry.go:498] Detected object tsf-tpa/migrate-db-58x72 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:52.661350 3681 obj_retry.go:498] Detected object tsf-tpa/migrate-db-58x72 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:52.794874 3681 obj_retry.go:492] Detected object tsf-tpa/migrate-db-58x72 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:52.794902 3681 obj_retry.go:492] Detected object tsf-tpa/migrate-db-58x72 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:55.691166 3681 obj_retry.go:498] Detected object tsf-tpa/create-importers-ldjg5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:55.691821 3681 obj_retry.go:498] Detected object tsf-tpa/create-importers-ldjg5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:49:56.003834 3681 obj_retry.go:492] Detected object tsf-tpa/create-importers-ldjg5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:49:56.003858 3681 obj_retry.go:492] Detected object tsf-tpa/create-importers-ldjg5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:50:00.171916 3681 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29607530-gw94z for pod on switch ip-10-0-2-177.ec2.internal I0417 18:50:00.172018 3681 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z I0417 18:50:00.183794 3681 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z] pod update took 11.80074ms I0417 18:50:00.183827 3681 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z] addLogicalPort annotation time took 11.837325ms I0417 18:50:00.185539 3681 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z] addLogicalPort took 13.643722ms, libovsdb time 1.130033ms I0417 18:50:00.537119 3681 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z 433bba79fc57207539b0519a7a1e8bac472f7151af5d469b85c72c6f98453081 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z 433bba79fc57207539b0519a7a1e8bac472f7151af5d469b85c72c6f98453081 network default NAD default NAD key ] I0417 18:50:00.546481 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29607530-gw94z, hostIfaceName: 433bba79fc57207, network: default, NAD default, SandboxID: "433bba79fc57207539b0519a7a1e8bac472f7151af5d469b85c72c6f98453081", PCI device ID: , UID: "a45ca4f2-4ce1-4ac5-995b-b3528949a469", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0417 18:50:00.602229 3681 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z 433bba79fc57207539b0519a7a1e8bac472f7151af5d469b85c72c6f98453081 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z 433bba79fc57207539b0519a7a1e8bac472f7151af5d469b85c72c6f98453081 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:83:d2:e1:00:ec\",\"name\":\"433bba79fc57207\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/90011ce2-0562-4bed-9bc7-e346f65f7843\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:50:03.846894 3681 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z 433bba79fc57207539b0519a7a1e8bac472f7151af5d469b85c72c6f98453081 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z 433bba79fc57207539b0519a7a1e8bac472f7151af5d469b85c72c6f98453081 network default NAD default NAD key ] I0417 18:50:03.937375 3681 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z 433bba79fc57207539b0519a7a1e8bac472f7151af5d469b85c72c6f98453081 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z 433bba79fc57207539b0519a7a1e8bac472f7151af5d469b85c72c6f98453081 network default NAD default NAD key default], result "{}", err I0417 18:50:03.988784 3681 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:50:03.988822 3681 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z I0417 18:50:03.990243 3681 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z, ips: 10.128.0.47 I0417 18:50:03.990275 3681 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:50:04.859554 3681 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:50:04.859584 3681 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607530-gw94z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0417 18:50:05.755519 3681 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0417 18:50:14.792434 3681 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-gxnqd-j92xv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:50:14.793058 3681 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-gxnqd-j92xv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:50:15.683119 3681 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-gxnqd-j92xv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:50:15.683146 3681 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-gxnqd-j92xv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0417 18:50:22.705129 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0417 18:50:24.053798 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0417 18:50:34.009608 3681 cni.go:452] [tsf-tpa/test-tsf-tpa aa62b2e320e4739cfb4d29818d1fe5c58280a0d5659fd1b24108bbe43eb53416 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/test-tsf-tpa aa62b2e320e4739cfb4d29818d1fe5c58280a0d5659fd1b24108bbe43eb53416 network default NAD default NAD key ] I0417 18:50:34.103001 3681 cni.go:473] [tsf-tpa/test-tsf-tpa aa62b2e320e4739cfb4d29818d1fe5c58280a0d5659fd1b24108bbe43eb53416 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/test-tsf-tpa aa62b2e320e4739cfb4d29818d1fe5c58280a0d5659fd1b24108bbe43eb53416 network default NAD default NAD key default], result "{}", err I0417 18:50:34.155353 3681 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:50:34.155393 3681 pods.go:176] Deleting pod: tsf-tpa/test-tsf-tpa I0417 18:50:34.156832 3681 pods.go:236] Attempting to release IPs for pod: tsf-tpa/test-tsf-tpa, ips: 10.128.0.46 I0417 18:50:34.156862 3681 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:50:34.191875 3681 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:50:34.191904 3681 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:50:34.197764 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0417 18:50:34.197793 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0417 18:50:34.997638 3681 namespace.go:98] [build-service] adding namespace I0417 18:50:34.997764 3681 namespace.go:98] [integration-service] adding namespace I0417 18:50:34.997800 3681 namespace.go:98] [default-tenant] adding namespace I0417 18:50:34.997879 3681 namespace.go:98] [image-controller] adding namespace I0417 18:50:34.998518 3681 namespace.go:102] [build-service] adding namespace took 861.151µs I0417 18:50:34.999304 3681 namespace.go:102] [integration-service] adding namespace took 1.522145ms I0417 18:50:34.999924 3681 namespace.go:102] [default-tenant] adding namespace took 2.109333ms I0417 18:50:35.000520 3681 namespace.go:102] [image-controller] adding namespace took 2.632507ms I0417 18:50:35.080860 3681 namespace.go:142] [build-service] updating namespace I0417 18:50:35.098806 3681 namespace.go:142] [build-service] updating namespace I0417 18:50:35.107576 3681 namespace.go:142] [integration-service] updating namespace I0417 18:50:35.122799 3681 namespace.go:142] [integration-service] updating namespace I0417 18:50:35.140609 3681 namespace.go:142] [default-tenant] updating namespace I0417 18:50:35.166552 3681 namespace.go:142] [image-controller] updating namespace I0417 18:50:35.166592 3681 namespace.go:142] [default-tenant] updating namespace I0417 18:50:35.200626 3681 namespace.go:142] [image-controller] updating namespace I0417 18:50:35.289751 3681 namespace.go:142] [build-service] updating namespace I0417 18:50:35.336580 3681 namespace.go:142] [integration-service] updating namespace I0417 18:50:35.394557 3681 namespace.go:98] [release-service] adding namespace I0417 18:50:35.395559 3681 namespace.go:102] [release-service] adding namespace took 977.78µs I0417 18:50:35.443377 3681 namespace.go:142] [release-service] updating namespace I0417 18:50:35.454283 3681 namespace.go:142] [release-service] updating namespace I0417 18:50:35.480306 3681 namespace.go:142] [konflux-ui] updating namespace I0417 18:50:35.663337 3681 namespace.go:98] [konflux-info] adding namespace I0417 18:50:35.664318 3681 namespace.go:102] [konflux-info] adding namespace took 959.185µs I0417 18:50:35.735539 3681 namespace.go:142] [konflux-info] updating namespace I0417 18:50:35.739565 3681 namespace.go:142] [konflux-info] updating namespace I0417 18:50:35.764453 3681 namespace.go:98] [namespace-lister] adding namespace I0417 18:50:35.765405 3681 namespace.go:102] [namespace-lister] adding namespace took 930.793µs I0417 18:50:35.803658 3681 namespace.go:142] [build-service] updating namespace I0417 18:50:35.803746 3681 namespace.go:142] [namespace-lister] updating namespace I0417 18:50:35.858466 3681 namespace.go:142] [namespace-lister] updating namespace I0417 18:50:35.904467 3681 namespace.go:98] [enterprise-contract-service] adding namespace I0417 18:50:35.905360 3681 namespace.go:102] [enterprise-contract-service] adding namespace took 860.375µs I0417 18:50:35.966791 3681 namespace.go:142] [enterprise-contract-service] updating namespace I0417 18:50:36.019534 3681 namespace.go:142] [image-controller] updating namespace I0417 18:50:36.032379 3681 namespace.go:142] [enterprise-contract-service] updating namespace I0417 18:50:36.076326 3681 namespace.go:142] [default-tenant] updating namespace I0417 18:50:36.408391 3681 namespace.go:142] [image-controller] updating namespace I0417 18:50:36.610505 3681 namespace.go:142] [integration-service] updating namespace I0417 18:50:36.751354 3681 namespace.go:142] [build-service] updating namespace I0417 18:50:36.859508 3681 namespace.go:142] [default-tenant] updating namespace I0417 18:50:36.861232 3681 namespace.go:142] [image-controller] updating namespace I0417 18:50:36.861358 3681 namespace.go:142] [integration-service] updating namespace I0417 18:50:37.099859 3681 namespace.go:142] [default-tenant] updating namespace I0417 18:50:37.161497 3681 namespace.go:142] [build-service] updating namespace I0417 18:50:37.200878 3681 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-67655c4674-vtbpb] creating logical port namespace-lister_namespace-lister-67655c4674-vtbpb for pod on switch ip-10-0-2-177.ec2.internal I0417 18:50:37.200991 3681 kube.go:256] Updating pod namespace-lister/namespace-lister-67655c4674-vtbpb I0417 18:50:37.283609 3681 pod.go:62] [namespace-lister/namespace-lister-67655c4674-vtbpb] pod update took 82.655049ms I0417 18:50:37.283638 3681 base_network_controller_pods.go:951] [default/namespace-lister/namespace-lister-67655c4674-vtbpb] addLogicalPort annotation time took 82.690217ms I0417 18:50:37.285055 3681 pods.go:271] [namespace-lister/namespace-lister-67655c4674-vtbpb] addLogicalPort took 84.193332ms, libovsdb time 944.223µs I0417 18:50:37.293182 3681 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0417 18:50:37.293730 3681 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0417 18:50:37.293747 3681 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0417 18:50:37.296822 3681 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0417 18:50:37.365281 3681 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0417 18:50:37.365309 3681 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0417 18:50:37.366743 3681 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0417 18:50:37.436467 3681 namespace.go:142] [default-tenant] updating namespace I0417 18:50:37.524472 3681 namespace.go:98] [segment-bridge] adding namespace I0417 18:50:37.525295 3681 namespace.go:102] [segment-bridge] adding namespace took 803.206µs I0417 18:50:37.533505 3681 namespace.go:142] [image-controller] updating namespace I0417 18:50:37.556294 3681 namespace.go:142] [segment-bridge] updating namespace I0417 18:50:37.621643 3681 namespace.go:142] [segment-bridge] updating namespace I0417 18:50:37.671415 3681 namespace.go:142] [integration-service] updating namespace I0417 18:50:37.892855 3681 base_network_controller_pods.go:487] [default/konflux-ui/dex-5f56fb9459-6cfnw] creating logical port konflux-ui_dex-5f56fb9459-6cfnw for pod on switch ip-10-0-2-177.ec2.internal I0417 18:50:37.892959 3681 kube.go:256] Updating pod konflux-ui/dex-5f56fb9459-6cfnw I0417 18:50:37.959489 3681 kube.go:256] Updating pod konflux-ui/dex-5f56fb9459-6cfnw I0417 18:50:37.963860 3681 base_network_controller_pods.go:487] [default/konflux-ui/proxy-776599d8ff-zdgsj] creating logical port konflux-ui_proxy-776599d8ff-zdgsj for pod on switch ip-10-0-2-177.ec2.internal I0417 18:50:37.964036 3681 kube.go:256] Updating pod konflux-ui/proxy-776599d8ff-zdgsj I0417 18:50:38.006315 3681 pod.go:62] [konflux-ui/dex-5f56fb9459-6cfnw] pod update took 113.384765ms I0417 18:50:38.006339 3681 base_network_controller_pods.go:951] [default/konflux-ui/dex-5f56fb9459-6cfnw] addLogicalPort annotation time took 113.413815ms I0417 18:50:38.007628 3681 pods.go:271] [konflux-ui/dex-5f56fb9459-6cfnw] addLogicalPort took 114.794867ms, libovsdb time 844.292µs I0417 18:50:38.073498 3681 kube.go:256] Updating pod konflux-ui/proxy-776599d8ff-zdgsj I0417 18:50:38.132537 3681 pod.go:62] [konflux-ui/proxy-776599d8ff-zdgsj] pod update took 168.563993ms I0417 18:50:38.132560 3681 base_network_controller_pods.go:951] [default/konflux-ui/proxy-776599d8ff-zdgsj] addLogicalPort annotation time took 168.592637ms I0417 18:50:38.133976 3681 pods.go:271] [konflux-ui/proxy-776599d8ff-zdgsj] addLogicalPort took 170.132784ms, libovsdb time 943.837µs I0417 18:50:38.165465 3681 cni.go:452] [namespace-lister/namespace-lister-67655c4674-vtbpb 6a6d41bac15e44bf80c86448a88dde21308916327dc0f01661aef238d19f58f5 network default NAD default NAD key ] ADD starting CNI request [namespace-lister/namespace-lister-67655c4674-vtbpb 6a6d41bac15e44bf80c86448a88dde21308916327dc0f01661aef238d19f58f5 network default NAD default NAD key ] I0417 18:50:38.175519 3681 helper_linux.go:556] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-67655c4674-vtbpb, hostIfaceName: 6a6d41bac15e44b, network: default, NAD default, SandboxID: "6a6d41bac15e44bf80c86448a88dde21308916327dc0f01661aef238d19f58f5", PCI device ID: , UID: "1230da9f-3f59-4555-a96d-f3172514b3f0", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0417 18:50:38.231042 3681 cni.go:473] [namespace-lister/namespace-lister-67655c4674-vtbpb 6a6d41bac15e44bf80c86448a88dde21308916327dc0f01661aef238d19f58f5 network default NAD default NAD key default] ADD finished CNI request [namespace-lister/namespace-lister-67655c4674-vtbpb 6a6d41bac15e44bf80c86448a88dde21308916327dc0f01661aef238d19f58f5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"fa:43:92:5f:ce:1a\",\"name\":\"6a6d41bac15e44b\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b1597751-4d64-49ba-907f-b2a731a3d918\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:50:38.650863 3681 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7cbf488fcf-6nrz6] creating logical port build-service_build-service-controller-manager-7cbf488fcf-6nrz6 for pod on switch ip-10-0-2-177.ec2.internal I0417 18:50:38.650968 3681 kube.go:256] Updating pod build-service/build-service-controller-manager-7cbf488fcf-6nrz6 I0417 18:50:38.668354 3681 pod.go:62] [build-service/build-service-controller-manager-7cbf488fcf-6nrz6] pod update took 17.41309ms I0417 18:50:38.668378 3681 base_network_controller_pods.go:951] [default/build-service/build-service-controller-manager-7cbf488fcf-6nrz6] addLogicalPort annotation time took 17.443236ms I0417 18:50:38.669805 3681 pods.go:271] [build-service/build-service-controller-manager-7cbf488fcf-6nrz6] addLogicalPort took 18.960361ms, libovsdb time 937.751µs I0417 18:50:38.883389 3681 namespace.go:142] [enterprise-contract-service] updating namespace I0417 18:50:39.016109 3681 cni.go:452] [build-service/build-service-controller-manager-7cbf488fcf-6nrz6 87b91a88179011d5b1b1f83e0eeb3f4b916bb2972faf18f5479931251ddf8c84 network default NAD default NAD key ] ADD starting CNI request [build-service/build-service-controller-manager-7cbf488fcf-6nrz6 87b91a88179011d5b1b1f83e0eeb3f4b916bb2972faf18f5479931251ddf8c84 network default NAD default NAD key ] I0417 18:50:39.026373 3681 helper_linux.go:556] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7cbf488fcf-6nrz6, hostIfaceName: 87b91a88179011d, network: default, NAD default, SandboxID: "87b91a88179011d5b1b1f83e0eeb3f4b916bb2972faf18f5479931251ddf8c84", PCI device ID: , UID: "417205dd-d5a4-46ae-b1d7-436ad96cfdd2", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0417 18:50:39.085793 3681 namespace.go:142] [konflux-info] updating namespace I0417 18:50:39.091658 3681 cni.go:473] [build-service/build-service-controller-manager-7cbf488fcf-6nrz6 87b91a88179011d5b1b1f83e0eeb3f4b916bb2972faf18f5479931251ddf8c84 network default NAD default NAD key default] ADD finished CNI request [build-service/build-service-controller-manager-7cbf488fcf-6nrz6 87b91a88179011d5b1b1f83e0eeb3f4b916bb2972faf18f5479931251ddf8c84 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:ef:91:1c:95:98\",\"name\":\"87b91a88179011d\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2bbd9d6e-b129-41a7-9e35-f2f2da85d1ca\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:50:39.304534 3681 namespace.go:142] [namespace-lister] updating namespace I0417 18:50:39.493441 3681 namespace.go:142] [release-service] updating namespace I0417 18:50:39.687627 3681 namespace.go:142] [segment-bridge] updating namespace I0417 18:50:39.802875 3681 namespace.go:142] [enterprise-contract-service] updating namespace I0417 18:50:39.821599 3681 namespace.go:142] [konflux-info] updating namespace I0417 18:50:39.860485 3681 namespace.go:142] [namespace-lister] updating namespace I0417 18:50:39.877529 3681 namespace.go:142] [release-service] updating namespace I0417 18:50:39.913522 3681 namespace.go:142] [segment-bridge] updating namespace I0417 18:50:39.937197 3681 namespace.go:142] [enterprise-contract-service] updating namespace I0417 18:50:40.010469 3681 namespace.go:142] [konflux-info] updating namespace I0417 18:50:40.122887 3681 namespace.go:142] [namespace-lister] updating namespace I0417 18:50:40.208466 3681 namespace.go:142] [release-service] updating namespace I0417 18:50:40.315413 3681 namespace.go:142] [segment-bridge] updating namespace W0417 18:50:41.059188 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister W0417 18:50:41.087344 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister W0417 18:50:43.070246 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0417 18:50:43.857132 3681 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0417 18:50:45.732803 3681 cni.go:452] [konflux-ui/dex-5f56fb9459-6cfnw fc1f2d6c822356bcd7481250c70778969fa7203010cef306bab57f94159cc806 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/dex-5f56fb9459-6cfnw fc1f2d6c822356bcd7481250c70778969fa7203010cef306bab57f94159cc806 network default NAD default NAD key ] I0417 18:50:45.742387 3681 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: dex-5f56fb9459-6cfnw, hostIfaceName: fc1f2d6c822356b, network: default, NAD default, SandboxID: "fc1f2d6c822356bcd7481250c70778969fa7203010cef306bab57f94159cc806", PCI device ID: , UID: "68034278-303b-430e-8f00-6fde3db6f7b7", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0417 18:50:45.802835 3681 cni.go:473] [konflux-ui/dex-5f56fb9459-6cfnw fc1f2d6c822356bcd7481250c70778969fa7203010cef306bab57f94159cc806 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/dex-5f56fb9459-6cfnw fc1f2d6c822356bcd7481250c70778969fa7203010cef306bab57f94159cc806 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:0d:00:ad:47:a9\",\"name\":\"fc1f2d6c822356b\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/68b59167-3778-45da-949d-ee2735c21763\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 18:50:45.821512 3681 cni.go:452] [konflux-ui/proxy-776599d8ff-zdgsj 5d3fd58536d02ae4d4c257cb0328216aa6a3b9d284d9d2682f292a7f26fa292a network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-776599d8ff-zdgsj 5d3fd58536d02ae4d4c257cb0328216aa6a3b9d284d9d2682f292a7f26fa292a network default NAD default NAD key ] I0417 18:50:45.831374 3681 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-776599d8ff-zdgsj, hostIfaceName: 5d3fd58536d02ae, network: default, NAD default, SandboxID: "5d3fd58536d02ae4d4c257cb0328216aa6a3b9d284d9d2682f292a7f26fa292a", PCI device ID: , UID: "76cd8ff7-26f9-4ab6-b7c1-f087378447b7", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0417 18:50:45.887573 3681 cni.go:473] [konflux-ui/proxy-776599d8ff-zdgsj 5d3fd58536d02ae4d4c257cb0328216aa6a3b9d284d9d2682f292a7f26fa292a network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-776599d8ff-zdgsj 5d3fd58536d02ae4d4c257cb0328216aa6a3b9d284d9d2682f292a7f26fa292a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:68:7e:8d:4c:e0\",\"name\":\"5d3fd58536d02ae\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/806caa44-5b30-4e14-be3c-c548f1ebae9a\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0417 18:50:48.097525 3681 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0417 18:50:52.134227 3681 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0417 18:50:52.134252 3681 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0417 18:50:53.947728 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0417 18:50:53.948647 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0417 18:50:53.965064 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0417 18:50:53.967931 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0417 18:50:53.993946 3681 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0417 18:50:53.993972 3681 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0417 18:50:54.016671 3681 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0417 18:50:54.016706 3681 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0417 18:50:54.077042 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0417 18:51:03.135767 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex I0417 18:51:05.339888 3681 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-jx4qd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:51:05.340476 3681 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-jx4qd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:51:06.030967 3681 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-jx4qd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:51:06.031003 3681 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-jx4qd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:51:06.033979 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-jx4qd I0417 18:51:06.033998 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-jx4qd I0417 18:51:10.065996 3681 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 W0417 18:51:33.221189 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0417 18:51:33.221214 3681 util.go:959] Failed to build node endpoints for node ip-10-0-2-177.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0417 18:52:03.211824 3681 cni.go:452] [konflux-ui/proxy-776599d8ff-zdgsj 5d3fd58536d02ae4d4c257cb0328216aa6a3b9d284d9d2682f292a7f26fa292a network default NAD default NAD key ] DEL starting CNI request [konflux-ui/proxy-776599d8ff-zdgsj 5d3fd58536d02ae4d4c257cb0328216aa6a3b9d284d9d2682f292a7f26fa292a network default NAD default NAD key ] I0417 18:52:03.306128 3681 cni.go:473] [konflux-ui/proxy-776599d8ff-zdgsj 5d3fd58536d02ae4d4c257cb0328216aa6a3b9d284d9d2682f292a7f26fa292a network default NAD default NAD key default] DEL finished CNI request [konflux-ui/proxy-776599d8ff-zdgsj 5d3fd58536d02ae4d4c257cb0328216aa6a3b9d284d9d2682f292a7f26fa292a network default NAD default NAD key default], result "{}", err I0417 18:52:03.368123 3681 obj_retry.go:498] Detected object konflux-ui/proxy-776599d8ff-zdgsj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:52:03.368161 3681 pods.go:176] Deleting pod: konflux-ui/proxy-776599d8ff-zdgsj I0417 18:52:03.369554 3681 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-776599d8ff-zdgsj, ips: 10.128.0.50 I0417 18:52:03.369587 3681 obj_retry.go:498] Detected object konflux-ui/proxy-776599d8ff-zdgsj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:52:03.544162 3681 obj_retry.go:492] Detected object konflux-ui/proxy-776599d8ff-zdgsj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:52:03.544193 3681 obj_retry.go:492] Detected object konflux-ui/proxy-776599d8ff-zdgsj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:52:03.550400 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-776599d8ff-zdgsj I0417 18:52:03.550441 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-776599d8ff-zdgsj I0417 18:52:08.562243 3681 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 18:52:08.562851 3681 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 18:52:08.579292 3681 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:52:08.579316 3681 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:52:08.585545 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0417 18:52:08.585563 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0417 18:52:14.945659 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=154 I0417 18:52:42.121791 3681 namespace.go:98] [default-managed-tenant-xa4b3] adding namespace I0417 18:52:42.122894 3681 namespace.go:102] [default-managed-tenant-xa4b3] adding namespace took 1.077044ms I0417 18:52:42.184712 3681 namespace.go:142] [default-managed-tenant-xa4b3] updating namespace I0417 18:52:42.202806 3681 namespace.go:142] [default-managed-tenant-xa4b3] updating namespace I0417 18:52:43.144462 3681 namespace.go:142] [default-managed-tenant-xa4b3] updating namespace I0417 18:52:43.275892 3681 namespace.go:142] [default-managed-tenant-xa4b3] updating namespace I0417 18:52:43.373805 3681 namespace.go:142] [default-managed-tenant-xa4b3] updating namespace I0417 18:53:27.059579 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=9 I0417 18:53:39.953946 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=375 I0417 18:53:47.942875 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=70 I0417 18:54:44.943641 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0417 18:55:15.033124 3681 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0417 18:55:37.944998 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=64 I0417 18:55:41.949868 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=75 I0417 18:56:04.080149 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0417 18:56:05.036934 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0417 18:56:09.041450 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=10 I0417 18:56:17.047063 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0417 18:56:34.071525 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=10 I0417 18:57:14.949139 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=5 I0417 18:57:56.069511 3681 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0417 18:58:30.057806 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0417 18:58:45.033013 3681 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=10 I0417 18:59:49.046941 3681 obj_retry.go:492] Detected object tsf-tpa/create-db-dlckp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:59:49.046989 3681 obj_retry.go:492] Detected object tsf-tpa/create-db-dlckp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:59:49.050992 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-dlckp I0417 18:59:49.051020 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-dlckp I0417 18:59:52.029042 3681 obj_retry.go:492] Detected object tsf-tpa/migrate-db-58x72 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:59:52.029099 3681 obj_retry.go:492] Detected object tsf-tpa/migrate-db-58x72 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:59:52.033574 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-58x72 I0417 18:59:52.033605 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-58x72 I0417 18:59:56.026811 3681 obj_retry.go:492] Detected object tsf-tpa/create-importers-ldjg5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:59:56.026867 3681 obj_retry.go:492] Detected object tsf-tpa/create-importers-ldjg5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 18:59:56.031063 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-ldjg5 I0417 18:59:56.031084 3681 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-ldjg5 I0417 19:00:00.190510 3681 base_network_controller_pods.go:487] [default/segment-bridge/segment-bridge-29607540-lwn6v] creating logical port segment-bridge_segment-bridge-29607540-lwn6v for pod on switch ip-10-0-2-177.ec2.internal I0417 19:00:00.190648 3681 kube.go:256] Updating pod segment-bridge/segment-bridge-29607540-lwn6v I0417 19:00:00.216149 3681 pod.go:62] [segment-bridge/segment-bridge-29607540-lwn6v] pod update took 25.528417ms I0417 19:00:00.216175 3681 base_network_controller_pods.go:951] [default/segment-bridge/segment-bridge-29607540-lwn6v] addLogicalPort annotation time took 25.563932ms I0417 19:00:00.217599 3681 pods.go:271] [segment-bridge/segment-bridge-29607540-lwn6v] addLogicalPort took 27.103214ms, libovsdb time 937.643µs I0417 19:00:00.217679 3681 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29607540-x8gtn for pod on switch ip-10-0-2-177.ec2.internal I0417 19:00:00.217793 3681 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn I0417 19:00:00.231183 3681 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn] pod update took 13.427778ms I0417 19:00:00.231207 3681 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn] addLogicalPort annotation time took 13.458124ms I0417 19:00:00.232551 3681 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn] addLogicalPort took 14.876414ms, libovsdb time 809.558µs I0417 19:00:00.555170 3681 cni.go:452] [segment-bridge/segment-bridge-29607540-lwn6v e23fc937de9422869766555d4a77d0f5a5b548a885888119e3fb1406c3f382af network default NAD default NAD key ] ADD starting CNI request [segment-bridge/segment-bridge-29607540-lwn6v e23fc937de9422869766555d4a77d0f5a5b548a885888119e3fb1406c3f382af network default NAD default NAD key ] I0417 19:00:00.565264 3681 helper_linux.go:556] ConfigureOVS: namespace: segment-bridge, podName: segment-bridge-29607540-lwn6v, hostIfaceName: e23fc937de94228, network: default, NAD default, SandboxID: "e23fc937de9422869766555d4a77d0f5a5b548a885888119e3fb1406c3f382af", PCI device ID: , UID: "5c35dc61-178e-4ad1-b980-a999f83853a7", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0417 19:00:00.582288 3681 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn e0b2ceeedbd3d494fc531d19508345bbd355627e5d9724d14e144499c3f34b06 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn e0b2ceeedbd3d494fc531d19508345bbd355627e5d9724d14e144499c3f34b06 network default NAD default NAD key ] I0417 19:00:00.591842 3681 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29607540-x8gtn, hostIfaceName: e0b2ceeedbd3d49, network: default, NAD default, SandboxID: "e0b2ceeedbd3d494fc531d19508345bbd355627e5d9724d14e144499c3f34b06", PCI device ID: , UID: "64f31f36-fb1b-4257-a218-ca16d0da4e21", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] W0417 19:00:00.599316 3681 duration.go:461] Config duration recorder: measurement expired for pod/kube-system/ecr-credential-refresh-initial-4g8fb I0417 19:00:00.627674 3681 cni.go:473] [segment-bridge/segment-bridge-29607540-lwn6v e23fc937de9422869766555d4a77d0f5a5b548a885888119e3fb1406c3f382af network default NAD default NAD key default] ADD finished CNI request [segment-bridge/segment-bridge-29607540-lwn6v e23fc937de9422869766555d4a77d0f5a5b548a885888119e3fb1406c3f382af network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:e3:45:3e:5c:a1\",\"name\":\"e23fc937de94228\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/df1a1c9c-1ed4-47e3-9ded-c03911617527\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 19:00:00.650817 3681 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn e0b2ceeedbd3d494fc531d19508345bbd355627e5d9724d14e144499c3f34b06 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn e0b2ceeedbd3d494fc531d19508345bbd355627e5d9724d14e144499c3f34b06 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:ff:eb:5b:13:51\",\"name\":\"e0b2ceeedbd3d49\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/32b8c408-b64e-45e9-bf9c-1b0858dccc6e\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0417 19:00:04.062976 3681 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn e0b2ceeedbd3d494fc531d19508345bbd355627e5d9724d14e144499c3f34b06 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn e0b2ceeedbd3d494fc531d19508345bbd355627e5d9724d14e144499c3f34b06 network default NAD default NAD key ] I0417 19:00:04.158566 3681 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn e0b2ceeedbd3d494fc531d19508345bbd355627e5d9724d14e144499c3f34b06 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn e0b2ceeedbd3d494fc531d19508345bbd355627e5d9724d14e144499c3f34b06 network default NAD default NAD key default], result "{}", err I0417 19:00:04.210912 3681 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0417 19:00:04.210941 3681 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn I0417 19:00:04.212542 3681 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn, ips: 10.128.0.53 I0417 19:00:04.212565 3681 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0417 19:00:05.075042 3681 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 19:00:05.075073 3681 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29607540-x8gtn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0417 19:00:21.063264 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0417 19:00:52.036704 3681 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0417 19:01:53.945888 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=59 I0417 19:01:57.946797 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0417 19:02:28.083644 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 I0417 19:02:40.957557 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=37 I0417 19:03:05.050351 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0417 19:03:29.072855 3681 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=7 I0417 19:04:23.036886 3681 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0417 19:04:28.060888 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=5 I0417 19:04:29.952970 3681 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=8 I0417 19:04:45.074457 3681 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8