+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-9.ec2.internal ++ [[ -n ip-10-0-0-9.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-9.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0428 15:37:44.200192793 - disable conntrack on geneve port' I0428 15:37:44.200192793 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0428 15:37:44.222789086 - starting ovnkube-node + echo 'I0428 15:37:44.222789086 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-9.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-9.ec2.internal --init-node ip-10-0-0-9.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-9.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0428 15:37:44.264000 3817 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0428 15:37:44.264053 3817 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-6952a9a844.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0428 15:37:44.265880 3817 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0428 15:37:44.265906 3817 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0428 15:37:44.265950 3817 kube.go:431] Waiting for certificate I0428 15:37:44.266587 3817 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 15:37:44.288174 3817 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:44.288195 3817 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:44.290737 3817 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:44.294066 3817 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-764r2" I0428 15:37:44.306053 3817 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-764r2" I0428 15:37:44.306141 3817 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0428 15:37:45.266994 3817 kube.go:438] Certificate found I0428 15:37:45.267495 3817 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0428 15:37:45.268437 3817 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0428 15:37:45.321769 3817 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-29 15:32:44 +0000 UTC" deadline="2026-04-29 10:59:49.628229113 +0000 UTC" I0428 15:37:45.321806 3817 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="19h22m4.306434727s" I0428 15:37:45.332770 3817 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 15:37:45.349700 3817 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0428 15:37:45.349763 3817 server.go:101] MetricServer registers OVS metrics I0428 15:37:45.350053 3817 server.go:105] MetricServer registers OVN DB metrics I0428 15:37:45.350089 3817 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0428 15:37:45.350168 3817 factory.go:561] Starting watch factory I0428 15:37:45.350265 3817 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.350285 3817 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.350322 3817 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.350343 3817 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.350345 3817 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.350358 3817 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.350355 3817 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.350373 3817 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.350267 3817 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.350476 3817 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.351367 3817 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.351397 3817 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.354208 3817 controller_manager.go:370] Starting the ovnkube controller I0428 15:37:45.354219 3817 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-9.ec2.internal I0428 15:37:45.354295 3817 controller_manager.go:395] NBDB zone sync took: 65.953µs I0428 15:37:45.354337 3817 factory.go:561] Starting watch factory I0428 15:37:45.355116 3817 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0428 15:37:45.355145 3817 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0428 15:37:45.370282 3817 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.370723 3817 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.370869 3817 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0428 15:37:45.370923 3817 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0428 15:37:45.371242 3817 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.371277 3817 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.371350 3817 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.375793 3817 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0428 15:37:45.375869 3817 server.go:109] MetricServer registers OVN Controller metrics I0428 15:37:45.379485 3817 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0428 15:37:45.380609 3817 server.go:113] MetricServer registers OVN Northd metrics I0428 15:37:45.384460 3817 metrics.go:519] OVN Metrics Server starts to run ... I0428 15:37:45.456184 3817 factory.go:1977] *v1.Node informer cache synced successfully I0428 15:37:45.456207 3817 factory.go:1977] *v1.Service informer cache synced successfully I0428 15:37:45.456212 3817 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 15:37:45.456216 3817 factory.go:1977] *v1.Pod informer cache synced successfully I0428 15:37:45.456220 3817 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 15:37:45.456225 3817 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 15:37:45.456308 3817 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.456312 3817 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.456329 3817 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.456334 3817 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.458624 3817 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.458632 3817 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0428 15:37:45.459969 3817 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0428 15:37:45.459986 3817 factory.go:1977] *v1.Pod informer cache synced successfully I0428 15:37:45.459989 3817 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0428 15:37:45.459992 3817 factory.go:1977] *v1.Namespace informer cache synced successfully I0428 15:37:45.459995 3817 factory.go:1977] *v1.Node informer cache synced successfully I0428 15:37:45.459998 3817 factory.go:1977] *v1.Service informer cache synced successfully I0428 15:37:45.462178 3817 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 15:37:45.462189 3817 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 15:37:45.462236 3817 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.462247 3817 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.463752 3817 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.465123 3817 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0428 15:37:45.465134 3817 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0428 15:37:45.467261 3817 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 15:37:45.467331 3817 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.467349 3817 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.469056 3817 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.470543 3817 factory.go:1977] *v1.EgressIP informer cache synced successfully I0428 15:37:45.472723 3817 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 15:37:45.472786 3817 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.472805 3817 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.474591 3817 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.475955 3817 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0428 15:37:45.478166 3817 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 15:37:45.478226 3817 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.478243 3817 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.479914 3817 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.481189 3817 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0428 15:37:45.483335 3817 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 15:37:45.483390 3817 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.483403 3817 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.484964 3817 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.486325 3817 factory.go:1977] *v1.EgressService informer cache synced successfully I0428 15:37:45.488409 3817 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 15:37:45.488767 3817 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:37:45.488780 3817 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:37:45.490655 3817 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0428 15:37:45.491573 3817 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0428 15:37:45.494270 3817 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 15:37:45.494322 3817 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.494333 3817 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.494405 3817 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.494421 3817 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.496115 3817 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.496288 3817 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0428 15:37:45.497172 3817 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0428 15:37:45.499949 3817 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 15:37:45.499957 3817 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 15:37:45.499963 3817 factory.go:680] Watch Factory start up complete, took: 149.799936ms I0428 15:37:45.500048 3817 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0428 15:37:45.502343 3817 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0428 15:37:45.502352 3817 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0428 15:37:45.502359 3817 factory.go:680] Watch Factory start up complete, took: 148.022496ms I0428 15:37:45.502366 3817 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-0-9.ec2.internal" zone I0428 15:37:45.544010 3817 default_node_network_controller.go:710] Initializing the default node network controller I0428 15:37:45.625071 3817 udn_isolation.go:101] Starting UDN host isolation manager I0428 15:37:45.630995 3817 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0428 15:37:45.680749 3817 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0428 15:37:45.680828 3817 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0428 15:37:45.680842 3817 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0428 15:37:45.712328 3817 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0428 15:37:45.712389 3817 default_node_network_controller.go:809] Node ip-10-0-0-9.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0428 15:37:45.717178 3817 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.9"] k8s.ovn.org/zone-name:ip-10-0-0-9.ec2.internal] on node ip-10-0-0-9.ec2.internal I0428 15:37:45.736042 3817 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0428 15:37:45.742025 3817 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0428 15:37:45.742151 3817 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0428 15:37:45.742291 3817 gateway_init.go:263] Preparing Gateway I0428 15:37:45.742303 3817 gateway_shared_intf.go:1687] Creating new gateway I0428 15:37:45.753078 3817 controller_manager.go:429] Waiting for node in zone sync took: 250.700807ms I0428 15:37:45.756830 3817 controller_manager.go:611] Switching to transit router for layer2 networks I0428 15:37:45.756842 3817 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-9.ec2.internal I0428 15:37:45.771099 3817 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0428 15:37:45.776916 3817 controller_manager.go:290] SCTP support detected in OVN I0428 15:37:45.777421 3817 services_controller.go:69] Creating services controller for network=default I0428 15:37:45.777531 3817 udn_enabled_svc.go:80] Waiting for service informer to sync I0428 15:37:45.777602 3817 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0428 15:37:45.777629 3817 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0428 15:37:45.777638 3817 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0428 15:37:45.777646 3817 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 15:37:45.777807 3817 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0428 15:37:45.777856 3817 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0428 15:37:45.777916 3817 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 83.549µs I0428 15:37:45.777940 3817 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 71.837µs I0428 15:37:45.777955 3817 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 98.57µs I0428 15:37:45.777982 3817 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0428 15:37:45.777989 3817 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0428 15:37:45.777999 3817 default_network_controller.go:325] Starting the default network controller I0428 15:37:45.778008 3817 udn_enabled_svc.go:95] Performing full resync I0428 15:37:45.778081 3817 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 14.224µs I0428 15:37:45.778290 3817 udn_enabled_svc.go:99] Waiting for handler to sync I0428 15:37:45.778300 3817 udn_enabled_svc.go:104] Starting worker I0428 15:37:45.787126 3817 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0428 15:37:45.787193 3817 repair.go:33] Syncing exgw routes took 54.55µs I0428 15:37:45.787230 3817 default_network_controller.go:414] Starting all the Watchers... I0428 15:37:45.787409 3817 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0428 15:37:45.787421 3817 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0428 15:37:45.787429 3817 namespace.go:98] [openshift-console-user-settings] adding namespace I0428 15:37:45.787439 3817 namespace.go:98] [openshift-operators-redhat] adding namespace I0428 15:37:45.787448 3817 namespace.go:98] [openshift-network-console] adding namespace I0428 15:37:45.787453 3817 namespace.go:98] [openshift-monitoring] adding namespace I0428 15:37:45.787448 3817 namespace.go:98] [openshift-ingress-operator] adding namespace I0428 15:37:45.787467 3817 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0428 15:37:45.787471 3817 namespace.go:98] [openshift-backplane-tam] adding namespace I0428 15:37:45.787410 3817 namespace.go:98] [kube-system] adding namespace I0428 15:37:45.787478 3817 namespace.go:98] [openshift-cluster-version] adding namespace I0428 15:37:45.787519 3817 namespace.go:98] [openshift] adding namespace I0428 15:37:45.787410 3817 namespace.go:98] [openshift-customer-monitoring] adding namespace I0428 15:37:45.787532 3817 namespace.go:98] [openshift-backplane-csm] adding namespace I0428 15:37:45.787551 3817 namespace.go:98] [openshift-apiserver] adding namespace I0428 15:37:45.787986 3817 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 564.954µs I0428 15:37:45.788001 3817 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0428 15:37:45.788446 3817 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 439.876µs I0428 15:37:45.788459 3817 namespace.go:98] [openshift-infra] adding namespace I0428 15:37:45.789044 3817 namespace.go:102] [openshift-infra] adding namespace took 576.759µs I0428 15:37:45.789061 3817 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0428 15:37:45.789527 3817 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 2.096114ms I0428 15:37:45.789544 3817 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0428 15:37:45.789995 3817 namespace.go:102] [openshift-console-user-settings] adding namespace took 2.55591ms I0428 15:37:45.790011 3817 namespace.go:98] [openshift-operators] adding namespace I0428 15:37:45.790423 3817 namespace.go:102] [openshift-operators-redhat] adding namespace took 2.976707ms I0428 15:37:45.790441 3817 namespace.go:98] [openshift-service-ca-operator] adding namespace I0428 15:37:45.791048 3817 namespace.go:102] [openshift-network-console] adding namespace took 3.587163ms I0428 15:37:45.791063 3817 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0428 15:37:45.791804 3817 namespace.go:102] [openshift-monitoring] adding namespace took 4.339974ms I0428 15:37:45.791820 3817 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0428 15:37:45.792746 3817 namespace.go:102] [openshift-ingress-operator] adding namespace took 5.271197ms I0428 15:37:45.792760 3817 namespace.go:98] [openshift-dns] adding namespace I0428 15:37:45.793243 3817 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 5.761343ms I0428 15:37:45.793263 3817 namespace.go:98] [openshift-authentication] adding namespace I0428 15:37:45.793753 3817 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.269776ms I0428 15:37:45.793767 3817 namespace.go:98] [openshift-network-diagnostics] adding namespace I0428 15:37:45.794245 3817 namespace.go:102] [kube-system] adding namespace took 6.758368ms I0428 15:37:45.794261 3817 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0428 15:37:45.794872 3817 namespace.go:102] [openshift-cluster-version] adding namespace took 7.382318ms I0428 15:37:45.794892 3817 namespace.go:98] [openshift-must-gather-operator] adding namespace I0428 15:37:45.795332 3817 namespace.go:102] [openshift] adding namespace took 7.79996ms I0428 15:37:45.795359 3817 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0428 15:37:45.795894 3817 namespace.go:102] [openshift-customer-monitoring] adding namespace took 8.35904ms I0428 15:37:45.795912 3817 namespace.go:98] [openshift-image-registry] adding namespace I0428 15:37:45.796342 3817 namespace.go:102] [openshift-backplane-csm] adding namespace took 8.801725ms I0428 15:37:45.796356 3817 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0428 15:37:45.796950 3817 namespace.go:102] [openshift-apiserver] adding namespace took 9.358082ms I0428 15:37:45.796968 3817 namespace.go:98] [openshift-backplane-mobb] adding namespace I0428 15:37:45.797463 3817 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 8.395755ms I0428 15:37:45.797478 3817 namespace.go:98] [openshift-config-managed] adding namespace I0428 15:37:45.797936 3817 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 8.386274ms I0428 15:37:45.797953 3817 namespace.go:98] [openshift-dns-operator] adding namespace I0428 15:37:45.798323 3817 namespace.go:102] [openshift-operators] adding namespace took 8.306148ms I0428 15:37:45.798335 3817 namespace.go:98] [openshift-marketplace] adding namespace I0428 15:37:45.798739 3817 namespace.go:102] [openshift-service-ca-operator] adding namespace took 8.293256ms I0428 15:37:45.798751 3817 namespace.go:98] [openshift-node] adding namespace I0428 15:37:45.799102 3817 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 8.034472ms I0428 15:37:45.799114 3817 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0428 15:37:45.799574 3817 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.749939ms I0428 15:37:45.799586 3817 namespace.go:98] [openshift-kube-scheduler] adding namespace I0428 15:37:45.800025 3817 namespace.go:102] [openshift-dns] adding namespace took 7.25918ms I0428 15:37:45.800038 3817 namespace.go:98] [openshift-config-operator] adding namespace W0428 15:37:45.800391 3817 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-gzrrf: pod openshift-network-diagnostics/network-check-target-gzrrf: no pod IPs found W0428 15:37:45.800403 3817 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-w6lhl: pod openshift-network-diagnostics/network-check-target-w6lhl: no pod IPs found I0428 15:37:45.800423 3817 namespace.go:102] [openshift-authentication] adding namespace took 7.15108ms I0428 15:37:45.800438 3817 namespace.go:98] [openshift-insights] adding namespace I0428 15:37:45.800867 3817 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.095714ms I0428 15:37:45.800880 3817 namespace.go:98] [openshift-service-ca] adding namespace I0428 15:37:45.801215 3817 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.947447ms I0428 15:37:45.801231 3817 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0428 15:37:45.801806 3817 namespace.go:102] [openshift-must-gather-operator] adding namespace took 6.90574ms I0428 15:37:45.801826 3817 namespace.go:98] [openshift-etcd] adding namespace I0428 15:37:45.802290 3817 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 6.923983ms I0428 15:37:45.802305 3817 namespace.go:98] [openshift-package-operator] adding namespace I0428 15:37:45.802809 3817 namespace.go:102] [openshift-image-registry] adding namespace took 6.891922ms I0428 15:37:45.802822 3817 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0428 15:37:45.803276 3817 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 6.916097ms I0428 15:37:45.803288 3817 namespace.go:98] [openshift-host-network] adding namespace I0428 15:37:45.803678 3817 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.703213ms I0428 15:37:45.803694 3817 namespace.go:98] [openshift-kube-apiserver] adding namespace I0428 15:37:45.804244 3817 namespace.go:102] [openshift-config-managed] adding namespace took 6.762067ms I0428 15:37:45.804682 3817 namespace.go:102] [openshift-dns-operator] adding namespace took 6.724583ms I0428 15:37:45.804694 3817 namespace.go:98] [openshift-machine-config-operator] adding namespace I0428 15:37:45.805072 3817 namespace.go:102] [openshift-marketplace] adding namespace took 6.73291ms I0428 15:37:45.805084 3817 namespace.go:98] [openshift-console-operator] adding namespace I0428 15:37:45.805461 3817 namespace.go:102] [openshift-node] adding namespace took 6.704834ms I0428 15:37:45.805472 3817 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0428 15:37:45.805852 3817 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.733653ms I0428 15:37:45.805864 3817 namespace.go:98] [openshift-controller-manager] adding namespace I0428 15:37:45.806202 3817 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.609777ms I0428 15:37:45.806219 3817 namespace.go:98] [kube-node-lease] adding namespace I0428 15:37:45.806790 3817 namespace.go:102] [openshift-config-operator] adding namespace took 6.745859ms I0428 15:37:45.806804 3817 namespace.go:98] [openshift-apiserver-operator] adding namespace I0428 15:37:45.807337 3817 namespace.go:102] [openshift-insights] adding namespace took 6.892087ms I0428 15:37:45.807352 3817 namespace.go:98] [openshift-console] adding namespace I0428 15:37:45.807943 3817 namespace.go:102] [openshift-service-ca] adding namespace took 7.055782ms I0428 15:37:45.807960 3817 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0428 15:37:45.808406 3817 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.170181ms I0428 15:37:45.808421 3817 namespace.go:98] [default] adding namespace I0428 15:37:45.808814 3817 namespace.go:102] [openshift-etcd] adding namespace took 6.982533ms I0428 15:37:45.808823 3817 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0428 15:37:45.809409 3817 namespace.go:102] [openshift-package-operator] adding namespace took 7.096696ms I0428 15:37:45.809427 3817 namespace.go:98] [openshift-ingress-canary] adding namespace I0428 15:37:45.809914 3817 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 7.086638ms I0428 15:37:45.809926 3817 namespace.go:98] [openshift-ingress] adding namespace I0428 15:37:45.810513 3817 namespace.go:102] [openshift-host-network] adding namespace took 7.218512ms I0428 15:37:45.810529 3817 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0428 15:37:45.811024 3817 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.326186ms I0428 15:37:45.811033 3817 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0428 15:37:45.811508 3817 namespace.go:102] [openshift-machine-config-operator] adding namespace took 6.809824ms I0428 15:37:45.811516 3817 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0428 15:37:45.812080 3817 namespace.go:102] [openshift-console-operator] adding namespace took 6.990624ms I0428 15:37:45.812094 3817 namespace.go:98] [openshift-machine-api] adding namespace I0428 15:37:45.812518 3817 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.04152ms I0428 15:37:45.812530 3817 namespace.go:98] [openshift-network-operator] adding namespace I0428 15:37:45.812941 3817 namespace.go:102] [openshift-controller-manager] adding namespace took 7.071951ms I0428 15:37:45.812955 3817 namespace.go:98] [kube-public] adding namespace I0428 15:37:45.813326 3817 namespace.go:102] [kube-node-lease] adding namespace took 7.103105ms I0428 15:37:45.813339 3817 namespace.go:98] [openshift-multus] adding namespace I0428 15:37:45.813731 3817 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.921817ms I0428 15:37:45.813743 3817 namespace.go:98] [openshift-authentication-operator] adding namespace I0428 15:37:45.814095 3817 namespace.go:102] [openshift-console] adding namespace took 6.738512ms I0428 15:37:45.814106 3817 namespace.go:98] [openshift-network-node-identity] adding namespace I0428 15:37:45.814470 3817 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.505724ms I0428 15:37:45.814481 3817 namespace.go:98] [openshift-route-controller-manager] adding namespace I0428 15:37:45.814879 3817 namespace.go:102] [default] adding namespace took 6.454405ms I0428 15:37:45.814888 3817 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0428 15:37:45.815439 3817 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.612871ms I0428 15:37:45.815448 3817 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0428 15:37:45.815851 3817 namespace.go:102] [openshift-ingress-canary] adding namespace took 6.419094ms I0428 15:37:45.815859 3817 namespace.go:98] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] adding namespace I0428 15:37:45.816367 3817 namespace.go:102] [openshift-ingress] adding namespace took 6.437513ms I0428 15:37:45.816375 3817 namespace.go:98] [openshift-backplane] adding namespace I0428 15:37:45.816887 3817 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.350777ms I0428 15:37:45.816905 3817 namespace.go:98] [openshift-backplane-cse] adding namespace I0428 15:37:45.817494 3817 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 6.455259ms I0428 15:37:45.817509 3817 namespace.go:98] [openshift-backplane-srep] adding namespace I0428 15:37:45.817993 3817 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.473848ms I0428 15:37:45.818579 3817 namespace.go:102] [openshift-machine-api] adding namespace took 6.48095ms I0428 15:37:45.818591 3817 namespace.go:98] [dedicated-admin] adding namespace I0428 15:37:45.819077 3817 namespace.go:102] [openshift-network-operator] adding namespace took 6.540025ms I0428 15:37:45.819524 3817 namespace.go:102] [kube-public] adding namespace took 6.563581ms W0428 15:37:45.819529 3817 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-c85fc: pod openshift-multus/network-metrics-daemon-c85fc: no pod IPs found W0428 15:37:45.819539 3817 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-xdscj: pod openshift-multus/network-metrics-daemon-xdscj: no pod IPs found I0428 15:37:45.820156 3817 namespace.go:102] [openshift-multus] adding namespace took 6.812664ms I0428 15:37:45.820579 3817 namespace.go:102] [openshift-authentication-operator] adding namespace took 6.831401ms I0428 15:37:45.820590 3817 namespace.go:98] [openshift-backplane-cee] adding namespace I0428 15:37:45.820975 3817 namespace.go:102] [openshift-network-node-identity] adding namespace took 6.862921ms I0428 15:37:45.821364 3817 namespace.go:102] [openshift-route-controller-manager] adding namespace took 6.878173ms I0428 15:37:45.821375 3817 namespace.go:98] [openshift-config] adding namespace I0428 15:37:45.821750 3817 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 6.857656ms I0428 15:37:45.821764 3817 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0428 15:37:45.822118 3817 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.665247ms I0428 15:37:45.822129 3817 namespace.go:98] [openshift-logging] adding namespace I0428 15:37:45.822488 3817 namespace.go:102] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] adding namespace took 6.624054ms I0428 15:37:45.822857 3817 namespace.go:102] [openshift-backplane] adding namespace took 6.477429ms I0428 15:37:45.823215 3817 namespace.go:102] [openshift-backplane-cse] adding namespace took 6.306075ms I0428 15:37:45.823575 3817 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.062691ms I0428 15:37:45.823586 3817 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0428 15:37:45.824123 3817 namespace.go:102] [dedicated-admin] adding namespace took 5.52861ms I0428 15:37:45.824519 3817 namespace.go:102] [openshift-backplane-cee] adding namespace took 3.924469ms I0428 15:37:45.824912 3817 namespace.go:102] [openshift-config] adding namespace took 3.532436ms I0428 15:37:45.825283 3817 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 3.51533ms I0428 15:37:45.825687 3817 namespace.go:102] [openshift-logging] adding namespace took 3.554463ms I0428 15:37:45.826409 3817 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 2.818584ms W0428 15:37:45.826489 3817 master.go:458] Unable to parse local node ip-10-0-0-9.ec2.internal chassis-id annotation. Chassis may be removed during sync W0428 15:37:45.826498 3817 master.go:471] Unable to parse remote node ip-10-0-1-106.ec2.internal chassis-id annotation. Chassis may be removed during sync I0428 15:37:45.826503 3817 master.go:481] Removing stale chassis with ID/Name: cfce9355-11b8-49a0-bff5-8910b5b42803, hostname: ip-10-0-0-9.ec2.internal I0428 15:37:45.826928 3817 master.go:548] Adding or Updating local node "ip-10-0-0-9.ec2.internal" for network "default" I0428 15:37:45.826975 3817 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-106.ec2.internal, error: adding or updating remote node chassis ip-10-0-1-106.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-106.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-106.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-106.ec2.internal I0428 15:37:45.827615 3817 zone_ic_handler.go:195] Time taken to create transit switch: 335.613µs I0428 15:37:45.827627 3817 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-7.ec2.internal for the network default I0428 15:37:45.828445 3817 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-7.ec2.internal" on network "default" took: 1.176347ms I0428 15:37:45.830621 3817 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-9.ec2.internal for the network default I0428 15:37:45.830788 3817 zone_ic_handler.go:195] Time taken to create transit switch: 154.912µs I0428 15:37:45.831429 3817 default_network_controller.go:823] Node add failed for ip-10-0-0-9.ec2.internal, will try again later: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-9.ec2.internal, error getting gateway config for node ip-10-0-0-9.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-9.ec2.internal", failed to update chassis to local for local node ip-10-0-0-9.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-9.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-9.ec2.internal] E0428 15:37:45.831467 3817 obj_retry.go:575] Failed to create *v1.Node ip-10-0-0-9.ec2.internal, error: [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-9.ec2.internal, error getting gateway config for node ip-10-0-0-9.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-9.ec2.internal", failed to update chassis to local for local node ip-10-0-0-9.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-9.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-9.ec2.internal] I0428 15:37:45.831519 3817 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0428 15:37:45.831544 3817 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0428 15:37:45.831530 3817 event.go:377] Event(v1.ObjectReference{Kind:"Node", Namespace:"", Name:"ip-10-0-0-9.ec2.internal", UID:"0c5ac350-007e-4788-9298-74fddbd85d36", APIVersion:"v1", ResourceVersion:"9534", FieldPath:""}): type: 'Warning' reason: 'ErrorAddingResource' [k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-9.ec2.internal, error getting gateway config for node ip-10-0-0-9.ec2.internal: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-9.ec2.internal", failed to update chassis to local for local node ip-10-0-0-9.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-9.ec2.internal, error: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-9.ec2.internal] I0428 15:37:45.831574 3817 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0428 15:37:45.831578 3817 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-9.ec2.internal in network "default" I0428 15:37:45.831617 3817 node_tracker.go:235] Node ip-10-0-0-9.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-9.ec2.internal" I0428 15:37:45.831636 3817 node_tracker.go:169] Node ip-10-0-0-9.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.831656 3817 services_controller.go:574] Full service sync requested for network=default I0428 15:37:45.831671 3817 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:45.831687 3817 node_tracker.go:235] Node ip-10-0-1-106.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-106.ec2.internal" I0428 15:37:45.831700 3817 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.831707 3817 services_controller.go:574] Full service sync requested for network=default I0428 15:37:45.831716 3817 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-7.ec2.internal in network "default" I0428 15:37:45.831769 3817 node_tracker.go:169] Node ip-10-0-2-7.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.831778 3817 services_controller.go:574] Full service sync requested for network=default I0428 15:37:45.846510 3817 iptables.go:108] Creating table: filter chain: FORWARD I0428 15:37:45.852887 3817 iptables.go:108] Creating table: filter chain: OUTPUT I0428 15:37:45.858854 3817 gateway_shared_intf.go:1816] Gateway Creation Complete I0428 15:37:45.858867 3817 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-9.ec2.internal","mac-address":"12:81:6f:9c:31:1d","ip-addresses":["10.0.0.9/24"],"ip-address":"10.0.0.9/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:cfce9355-11b8-49a0-bff5-8910b5b42803 k8s.ovn.org/node-encap-ips:["10.0.0.9"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.9/24"} k8s.ovn.org/zone-name:ip-10-0-0-9.ec2.internal] on node ip-10-0-0-9.ec2.internal I0428 15:37:45.860083 3817 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:45.860102 3817 node_tracker.go:235] Node ip-10-0-1-106.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-106.ec2.internal" I0428 15:37:45.860116 3817 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.860130 3817 services_controller.go:574] Full service sync requested for network=default I0428 15:37:45.860159 3817 default_network_controller.go:1025] Node "ip-10-0-1-106.ec2.internal" in remote zone "ip-10-0-1-106.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:45.875716 3817 master.go:548] Adding or Updating local node "ip-10-0-0-9.ec2.internal" for network "default" I0428 15:37:45.875720 3817 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-9.ec2.internal in network "default" I0428 15:37:45.875846 3817 node_tracker.go:169] Node ip-10-0-0-9.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:45.875860 3817 services_controller.go:574] Full service sync requested for network=default I0428 15:37:45.879812 3817 default_node_network_controller.go:519] Node connection status = connected W0428 15:37:45.880648 3817 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-c85fc: pod openshift-multus/network-metrics-daemon-c85fc: no pod IPs found W0428 15:37:45.880669 3817 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-gzrrf: pod openshift-network-diagnostics/network-check-target-gzrrf: no pod IPs found I0428 15:37:45.881647 3817 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-9.ec2.internal for the network default I0428 15:37:45.881824 3817 zone_ic_handler.go:195] Time taken to create transit switch: 163.721µs I0428 15:37:45.893920 3817 default_network_controller.go:1025] Node "ip-10-0-1-106.ec2.internal" in remote zone "ip-10-0-1-106.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:45.931844 3817 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0428 15:37:45.931861 3817 services_controller.go:208] Setting up event handlers for services for network=default I0428 15:37:45.931925 3817 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0428 15:37:45.931961 3817 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0428 15:37:45.931977 3817 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0428 15:37:46.016939 3817 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:46.017000 3817 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:46.016998 3817 default_network_controller.go:1025] Node "ip-10-0-1-106.ec2.internal" in remote zone "ip-10-0-1-106.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0428 15:37:46.017014 3817 services_controller.go:574] Full service sync requested for network=default I0428 15:37:46.017917 3817 zone_ic_handler.go:195] Time taken to create transit switch: 252.198µs I0428 15:37:46.017931 3817 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-106.ec2.internal for the network default I0428 15:37:46.019147 3817 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-106.ec2.internal" on network "default" took: 1.492241ms I0428 15:37:46.032592 3817 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0428 15:37:46.032605 3817 repair.go:60] Starting repairing loop for services I0428 15:37:46.032683 3817 repair.go:131] Deleted 0 stale service LBs I0428 15:37:46.032698 3817 repair.go:137] Deleted 0 stale Chassis Template Vars I0428 15:37:46.032713 3817 repair.go:62] Finished repairing loop for services: 110.086µs I0428 15:37:46.032737 3817 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0428 15:37:46.032747 3817 services_controller.go:254] Starting workers for network=default I0428 15:37:46.033030 3817 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-xdscj, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-xdscj: suppressed error logged: pod openshift-multus/network-metrics-daemon-xdscj: no pod IPs found I0428 15:37:46.033043 3817 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-w6lhl, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-w6lhl: suppressed error logged: pod openshift-network-diagnostics/network-check-target-w6lhl: no pod IPs found I0428 15:37:46.033111 3817 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-gzrrf] creating logical port openshift-network-diagnostics_network-check-target-gzrrf for pod on switch ip-10-0-0-9.ec2.internal I0428 15:37:46.033185 3817 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-c85fc] creating logical port openshift-multus_network-metrics-daemon-c85fc for pod on switch ip-10-0-0-9.ec2.internal I0428 15:37:46.033275 3817 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-c85fc I0428 15:37:46.033361 3817 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-gzrrf W0428 15:37:46.035775 3817 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0428 15:37:46.035776 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0428 15:37:46.047245 3817 pod.go:62] [openshift-network-diagnostics/network-check-target-gzrrf] pod update took 14.018064ms I0428 15:37:46.047262 3817 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-gzrrf] addLogicalPort annotation time took 14.038177ms I0428 15:37:46.047908 3817 pod.go:62] [openshift-multus/network-metrics-daemon-c85fc] pod update took 14.680997ms I0428 15:37:46.047925 3817 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-c85fc] addLogicalPort annotation time took 14.701609ms I0428 15:37:46.048304 3817 pods.go:271] [openshift-network-diagnostics/network-check-target-gzrrf] addLogicalPort took 15.199938ms, libovsdb time 737.704µs I0428 15:37:46.048668 3817 pods.go:271] [openshift-multus/network-metrics-daemon-c85fc] addLogicalPort took 15.487335ms, libovsdb time 580.923µs I0428 15:37:46.049106 3817 admin_network_policy_controller.go:227] Starting controller default-network-controller I0428 15:37:46.049130 3817 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0428 15:37:46.049144 3817 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0428 15:37:46.049149 3817 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0428 15:37:46.049337 3817 repair.go:29] Repairing admin network policies took 184.227µs I0428 15:37:46.049498 3817 repair.go:92] Repairing baseline admin network policies took 150.033µs I0428 15:37:46.049988 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0428 15:37:46.050010 3817 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0428 15:37:46.051635 3817 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0428 15:37:46.051659 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0428 15:37:46.051671 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0428 15:37:46.053173 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0428 15:37:46.053193 3817 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0428 15:37:46.053201 3817 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0428 15:37:46.054675 3817 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0428 15:37:46.054691 3817 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0428 15:37:46.054700 3817 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0428 15:37:46.056578 3817 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0428 15:37:46.056596 3817 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0428 15:37:46.056603 3817 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0428 15:37:46.057778 3817 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0428 15:37:46.057798 3817 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0428 15:37:46.057810 3817 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0428 15:37:46.060223 3817 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0428 15:37:46.060241 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0428 15:37:46.060248 3817 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0428 15:37:46.060533 3817 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0428 15:37:46.060552 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0428 15:37:46.061233 3817 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0428 15:37:46.061247 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 15:37:46.061905 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0428 15:37:46.061926 3817 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0428 15:37:46.061933 3817 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0428 15:37:46.062219 3817 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0428 15:37:46.062237 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0428 15:37:46.062247 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0428 15:37:46.062733 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0428 15:37:46.062751 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0428 15:37:46.062759 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0428 15:37:46.064173 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0428 15:37:46.064189 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0428 15:37:46.064195 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0428 15:37:46.064458 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0428 15:37:46.064485 3817 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0428 15:37:46.064497 3817 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0428 15:37:46.064988 3817 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0428 15:37:46.065003 3817 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0428 15:37:46.065010 3817 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0428 15:37:46.065279 3817 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0428 15:37:46.065296 3817 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0428 15:37:46.065307 3817 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0428 15:37:46.066278 3817 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0428 15:37:46.066297 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0428 15:37:46.066304 3817 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0428 15:37:46.066946 3817 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0428 15:37:46.066959 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0428 15:37:46.066966 3817 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0428 15:37:46.067422 3817 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0428 15:37:46.067439 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0428 15:37:46.067448 3817 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0428 15:37:46.067898 3817 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0428 15:37:46.067917 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0428 15:37:46.067925 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0428 15:37:46.068399 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0428 15:37:46.068416 3817 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0428 15:37:46.068425 3817 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0428 15:37:46.069267 3817 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0428 15:37:46.069288 3817 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0428 15:37:46.069299 3817 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0428 15:37:46.070422 3817 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0428 15:37:46.070439 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0428 15:37:46.070446 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0428 15:37:46.070933 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0428 15:37:46.070949 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0428 15:37:46.070957 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0428 15:37:46.071406 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0428 15:37:46.071422 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0428 15:37:46.071429 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0428 15:37:46.071904 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0428 15:37:46.071918 3817 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0428 15:37:46.071926 3817 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0428 15:37:46.073781 3817 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0428 15:37:46.073796 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0428 15:37:46.073817 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0428 15:37:46.074446 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0428 15:37:46.074463 3817 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0428 15:37:46.074471 3817 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0428 15:37:46.075449 3817 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0428 15:37:46.075463 3817 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0428 15:37:46.075472 3817 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0428 15:37:46.076427 3817 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0428 15:37:46.076445 3817 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0428 15:37:46.076454 3817 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0428 15:37:46.077680 3817 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0428 15:37:46.082875 3817 egressfirewall.go:374] Starting EgressFirewall controller I0428 15:37:46.082916 3817 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0428 15:37:46.082921 3817 controller.go:133] Adding controller egress-firewall-controller event handlers I0428 15:37:46.082944 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0428 15:37:46.082955 3817 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0428 15:37:46.082944 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0428 15:37:46.082963 3817 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0428 15:37:46.083219 3817 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0428 15:37:46.083233 3817 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0428 15:37:46.083239 3817 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0428 15:37:46.083248 3817 egressqos.go:193] Setting up event handlers for EgressQoS I0428 15:37:46.083253 3817 egressfirewall.go:723] Syncing node "ip-10-0-0-9.ec2.internal" for egress firewall I0428 15:37:46.083261 3817 egressfirewall.go:723] Syncing node "ip-10-0-1-106.ec2.internal" for egress firewall I0428 15:37:46.083265 3817 egressfirewall.go:723] Syncing node "ip-10-0-2-7.ec2.internal" for egress firewall I0428 15:37:46.083824 3817 egressqos.go:245] Starting EgressQoS Controller I0428 15:37:46.083835 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0428 15:37:46.083841 3817 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0428 15:37:46.083847 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0428 15:37:46.083852 3817 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0428 15:37:46.083856 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0428 15:37:46.083861 3817 shared_informer.go:356] "Caches are synced" controller="egressqos" I0428 15:37:46.083865 3817 egressqos.go:259] Repairing EgressQoSes I0428 15:37:46.083868 3817 egressqos.go:399] Starting repairing loop for egressqos I0428 15:37:46.083935 3817 egressqos.go:401] Finished repairing loop for egressqos: 66.304µs I0428 15:37:46.083950 3817 egressservice_zone.go:132] Setting up event handlers for Egress Services I0428 15:37:46.083995 3817 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-9.ec2.internal I0428 15:37:46.084039 3817 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-9.ec2.internal : 45.551µs I0428 15:37:46.084055 3817 egressservice_zone.go:210] Starting Egress Services Controller I0428 15:37:46.084062 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 15:37:46.084069 3817 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 15:37:46.084075 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 15:37:46.084080 3817 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 15:37:46.084084 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 15:37:46.084089 3817 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 15:37:46.084093 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0428 15:37:46.084098 3817 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0428 15:37:46.084102 3817 egressservice_zone.go:228] Repairing Egress Services I0428 15:37:46.084356 3817 master_controller.go:88] Starting Admin Policy Based Route Controller I0428 15:37:46.084367 3817 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 15:37:46.084380 3817 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:37:46.084395 3817 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 17.731µs I0428 15:37:46.084409 3817 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:37:46.084415 3817 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 8.852µs I0428 15:37:46.084423 3817 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-7.ec2.internal I0428 15:37:46.084436 3817 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-7.ec2.internal: 13.948µs I0428 15:37:46.084475 3817 default_network_controller.go:585] Completing all the Watchers took 297.230297ms I0428 15:37:46.084491 3817 default_network_controller.go:589] Starting unidling controllers I0428 15:37:46.084506 3817 unidle.go:46] Registering OVN SB ControllerEvent handler I0428 15:37:46.084516 3817 unidle.go:63] Populating Initial ContollerEvent events I0428 15:37:46.084528 3817 unidle.go:79] Setting up event handlers for services I0428 15:37:46.084605 3817 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0428 15:37:46.380789 3817 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:46.880630 3817 default_node_network_controller.go:519] Node connection status = connected I0428 15:37:46.926806 3817 bridgeconfig.go:537] Gateway is ready I0428 15:37:46.926823 3817 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0428 15:37:46.938757 3817 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.9. Updating OVN encap IP. I0428 15:37:46.940962 3817 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0428 15:37:46.943874 3817 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.9 - it is already configured I0428 15:37:46.943889 3817 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.9/24:{} 172.20.0.1/32:{}]. Updating annotations. I0428 15:37:46.944136 3817 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.9/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-9.ec2.internal","mac-address":"12:81:6f:9c:31:1d","ip-addresses":["10.0.0.9/24"],"ip-address":"10.0.0.9/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:cfce9355-11b8-49a0-bff5-8910b5b42803 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.9/24"}] on node ip-10-0-0-9.ec2.internal I0428 15:37:46.960846 3817 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-9.ec2.internal in network "default" I0428 15:37:46.960886 3817 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:37:46.960894 3817 node_tracker.go:169] Node ip-10-0-0-9.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:46.960902 3817 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 20.513µs I0428 15:37:46.960907 3817 services_controller.go:574] Full service sync requested for network=default I0428 15:37:46.960886 3817 egressfirewall.go:723] Syncing node "ip-10-0-0-9.ec2.internal" for egress firewall I0428 15:37:46.960935 3817 master.go:548] Adding or Updating local node "ip-10-0-0-9.ec2.internal" for network "default" W0428 15:37:46.961078 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:37:46.961104 3817 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:46.961241 3817 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.0.9 I0428 15:37:46.961395 3817 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-9.ec2.internal I0428 15:37:46.965533 3817 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-9.ec2.internal. Updating no re-route policies I0428 15:37:46.965613 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-h8stv I0428 15:37:46.965621 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-9.ec2.internal I0428 15:37:46.965629 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-h8stv I0428 15:37:46.965634 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-9.ec2.internal I0428 15:37:46.965632 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-8phxm I0428 15:37:46.965638 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-d5t5b I0428 15:37:46.965647 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-8phxm I0428 15:37:46.965648 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-h8stv after 0 failed attempt(s) I0428 15:37:46.965651 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-9.ec2.internal after 0 failed attempt(s) I0428 15:37:46.965650 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-r47fq I0428 15:37:46.965652 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-d5t5b I0428 15:37:46.965661 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-gzrrf I0428 15:37:46.965664 3817 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-wlh92 I0428 15:37:46.965667 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-gzrrf I0428 15:37:46.965669 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-8phxm after 0 failed attempt(s) I0428 15:37:46.965671 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-d5t5b after 0 failed attempt(s) I0428 15:37:46.965672 3817 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-wlh92 I0428 15:37:46.965678 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-wxm7m I0428 15:37:46.965678 3817 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-9.ec2.internal I0428 15:37:46.965684 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-wxm7m I0428 15:37:46.965684 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-x2bb2 I0428 15:37:46.965686 3817 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-wlh92 after 0 failed attempt(s) I0428 15:37:46.965683 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-799wg I0428 15:37:46.965693 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-x2bb2 I0428 15:37:46.965689 3817 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-9.ec2.internal I0428 15:37:46.965702 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-wxm7m after 0 failed attempt(s) I0428 15:37:46.965705 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-799wg I0428 15:37:46.965711 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-x2bb2 after 0 failed attempt(s) I0428 15:37:46.965709 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-s8tbn I0428 15:37:46.965725 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-s8tbn I0428 15:37:46.965730 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-799wg after 0 failed attempt(s) I0428 15:37:46.965690 3817 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-c85fc I0428 15:37:46.965748 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-c85fc I0428 15:37:46.965749 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-s8tbn after 0 failed attempt(s) I0428 15:37:46.965714 3817 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-9.ec2.internal after 0 failed attempt(s) I0428 15:37:46.965692 3817 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-gzrrf] creating logical port openshift-network-diagnostics_network-check-target-gzrrf for pod on switch ip-10-0-0-9.ec2.internal I0428 15:37:46.965783 3817 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-c85fc] creating logical port openshift-multus_network-metrics-daemon-c85fc for pod on switch ip-10-0-0-9.ec2.internal I0428 15:37:46.965666 3817 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-r47fq I0428 15:37:46.965844 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-r47fq after 0 failed attempt(s) I0428 15:37:46.966406 3817 pods.go:271] [openshift-multus/network-metrics-daemon-c85fc] addLogicalPort took 631.582µs, libovsdb time 244.49µs I0428 15:37:46.966418 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-c85fc after 0 failed attempt(s) I0428 15:37:46.966475 3817 pods.go:271] [openshift-network-diagnostics/network-check-target-gzrrf] addLogicalPort took 791.585µs, libovsdb time 235.15µs I0428 15:37:46.966484 3817 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-gzrrf after 0 failed attempt(s) I0428 15:37:46.977270 3817 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0428 15:37:46.988991 3817 iptables.go:108] Creating table: mangle chain: OUTPUT I0428 15:37:46.992170 3817 iptables.go:108] Creating table: nat chain: OUTPUT I0428 15:37:47.040319 3817 iptables.go:108] Creating table: nat chain: PREROUTING I0428 15:37:47.086688 3817 iptables.go:108] Creating table: filter chain: FORWARD I0428 15:37:47.102476 3817 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0428 15:37:47.102623 3817 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0428 15:37:47.102683 3817 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0428 15:37:47.102770 3817 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0428 15:37:47.102864 3817 gateway_init.go:325] Gateway and management port readiness took 1.22727768s I0428 15:37:47.102898 3817 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0428 15:37:47.102971 3817 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0428 15:37:47.102987 3817 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0428 15:37:47.103000 3817 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0428 15:37:47.103085 3817 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0428 15:37:47.103168 3817 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 54.071µs I0428 15:37:47.103466 3817 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0428 15:37:47.103508 3817 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 21.061µs I0428 15:37:47.103520 3817 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 53.967µs I0428 15:37:47.103527 3817 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0428 15:37:47.103543 3817 nad_controller.go:379] [node-nad-controller NAD controller]: started I0428 15:37:47.103551 3817 default_node_network_controller.go:899] Starting the default node network controller I0428 15:37:47.103573 3817 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0428 15:37:47.103593 3817 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 14.845µs I0428 15:37:47.103764 3817 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0428 15:37:47.103775 3817 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0428 15:37:47.103805 3817 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b873b0 0xc0002ee3f0 0xc0002ee460 0xc0002ee4d0 0xc0002ee5b0 0xc0002ee620 0xc0002ee690 0xc000bea910 0xc0002ee700 0xc0002ee770 0xc0001a4280 0xc006559340 0xc0002ee7e0 map[0x2ff9080:0xc00259b920 0x2ff95c0:0xc00259baa0 0x2ffa2e0:0xc002a2e660 0x2ffa580:0xc00259b8c0 0x2ffad60:0xc00259b980 0x2ffb000:0xc00259b9e0 0x2ffb2a0:0xc00259ba40 0x2ffba80:0xc00259bb00 0x2ffbd20:0xc00259bc20 0x3026980:0xc001203140 0x3026d20:0xc0012031a0 0x3027800:0xc000bb9ec0 0x3029160:0xc001bfe000 0x30298a0:0xc0012032c0 0x302a720:0xc001bfe060] 0xc0000d3b90 0} I0428 15:37:47.103874 3817 gateway.go:151] Starting gateway service sync I0428 15:37:47.104336 3817 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0428 15:37:47.106378 3817 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0428 15:37:47.118201 3817 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0428 15:37:47.119320 3817 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0428 15:37:47.120356 3817 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0428 15:37:47.153282 3817 gateway.go:168] Gateway service sync done. Time taken: 49.395789ms I0428 15:37:47.153763 3817 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30931 I0428 15:37:47.154195 3817 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b873b0 0xc0002ee3f0 0xc0002ee460 0xc0002ee4d0 0xc0002ee5b0 0xc0002ee620 0xc0002ee690 0xc000bea910 0xc0002ee700 0xc0002ee770 0xc0001a4280 0xc006559340 0xc0002ee7e0 map[0x2ff9080:0xc00259b920 0x2ff95c0:0xc00259baa0 0x2ffa2e0:0xc002a2e660 0x2ffa580:0xc00259b8c0 0x2ffad60:0xc00259b980 0x2ffb000:0xc00259b9e0 0x2ffb2a0:0xc00259ba40 0x2ffba80:0xc00259bb00 0x2ffbd20:0xc00259bc20 0x3026980:0xc001203140 0x3026d20:0xc0012031a0 0x3027800:0xc000bb9ec0 0x3029160:0xc001bfe000 0x30298a0:0xc0012032c0 0x302a720:0xc001bfe060] 0xc0000d3b90 0} I0428 15:37:47.154809 3817 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000b873b0 0xc0002ee3f0 0xc0002ee460 0xc0002ee4d0 0xc0002ee5b0 0xc0002ee620 0xc0002ee690 0xc000bea910 0xc0002ee700 0xc0002ee770 0xc0001a4280 0xc006559340 0xc0002ee7e0 map[0x2ff9080:0xc00259b920 0x2ff95c0:0xc00259baa0 0x2ffa2e0:0xc002a2e660 0x2ffa580:0xc00259b8c0 0x2ffad60:0xc00259b980 0x2ffb000:0xc00259b9e0 0x2ffb2a0:0xc00259ba40 0x2ffba80:0xc00259bb00 0x2ffbd20:0xc00259bc20 0x3026980:0xc001203140 0x3026d20:0xc0012031a0 0x3027800:0xc000bb9ec0 0x3029160:0xc001bfe000 0x30298a0:0xc0012032c0 0x302a720:0xc001bfe060] 0xc0000d3b90 0} I0428 15:37:47.154870 3817 gateway.go:532] Reconciling gateway with updates I0428 15:37:47.160377 3817 gateway.go:339] Spawning Conntrack Rule Check Thread I0428 15:37:47.160504 3817 default_node_network_controller.go:944] Gateway and management port readiness took 56.718243ms I0428 15:37:47.160551 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:47.160574 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:47.160583 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:47.160590 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:47.160588 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:47.160592 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:47.160606 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:47.160611 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:47.160611 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:47.160612 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 15:37:47.160619 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 15:37:47.160575 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:47.160628 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:47.160625 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:47.160634 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:47.160640 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 15:37:47.160640 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 15:37:47.160644 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:47.160643 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:47.160651 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:47.160654 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:47.160623 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:47.160669 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:47.160674 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 15:37:47.160675 3817 port_claim.go:71] Svc openshift-ingress/router-default: port 32194 is already open, no action needed I0428 15:37:47.160678 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 15:37:47.160681 3817 port_claim.go:71] Svc openshift-ingress/router-default: port 31728 is already open, no action needed I0428 15:37:47.160673 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:47.160687 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 15:37:47.160687 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:47.160692 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 15:37:47.160685 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 15:37:47.160699 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 15:37:47.160698 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 15:37:47.160599 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 15:37:47.160708 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 15:37:47.160705 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:47.160713 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 15:37:47.160719 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:47.160722 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:47.160721 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:47.160730 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:47.160723 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:47.160734 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:47.160742 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:47.160649 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 15:37:47.160747 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:47.160694 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:47.160755 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:47.160758 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:47.160765 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:47.160766 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:47.160659 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:47.160767 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:47.160776 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:47.160778 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:47.160734 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:47.160788 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:47.160791 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:47.160723 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:47.160800 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:47.160804 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:47.160805 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:47.160813 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 15:37:47.160793 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:47.160774 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:47.160840 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:47.160847 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 15:37:47.160719 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:47.160855 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:47.160863 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:47.160866 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:47.160865 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:47.160873 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:47.160879 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:47.160881 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:47.160741 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:47.160895 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:47.160732 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:47.160940 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:47.160632 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:47.160965 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:47.160699 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:47.161057 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 15:37:47.160710 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:47.161075 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:47.160711 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:47.161141 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:47.160733 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:47.161190 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 15:37:47.160745 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:47.160744 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:47.161289 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:47.160761 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:47.160769 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:47.160776 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:47.161431 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:47.161437 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 15:37:47.160787 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:47.160841 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:47.161468 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:47.160855 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:47.160853 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:47.161517 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:47.161656 3817 node_ip_handler_linux.go:160] Node IP manager is running W0428 15:37:47.197502 3817 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 15:37:47.197776 3817 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0428 15:37:47.197929 3817 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0428 15:37:47.206622 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 15:37:47.206635 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 15:37:47.206626 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 15:37:47.206651 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 15:37:47.245178 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 15:37:47.245205 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 15:37:47.245212 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 15:37:47.245218 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 15:37:47.245223 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 15:37:47.245234 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 15:37:47.245245 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 15:37:47.245234 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 15:37:47.245253 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 15:37:47.245261 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 15:37:47.245266 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 15:37:47.245272 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 15:37:47.245277 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0428 15:37:47.284257 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 15:37:47.284285 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 15:37:47.284292 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 15:37:47.284298 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 15:37:47.284304 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 15:37:47.284310 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 15:37:47.284315 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 15:37:47.284321 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 15:37:47.284326 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 15:37:47.284332 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 15:37:47.284339 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 15:37:47.284345 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 15:37:47.284353 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 15:37:48.150020 3817 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-106.ec2.internal in network "default" I0428 15:37:48.150039 3817 egressfirewall.go:723] Syncing node "ip-10-0-1-106.ec2.internal" for egress firewall I0428 15:37:48.150059 3817 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:37:48.150067 3817 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-106.ec2.internal. Updating no re-route policies I0428 15:37:48.150072 3817 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 20.061µs I0428 15:37:48.150086 3817 node_tracker.go:169] Node ip-10-0-1-106.ec2.internal switch + router changed, syncing services in network "default" I0428 15:37:48.150116 3817 services_controller.go:574] Full service sync requested for network=default W0428 15:37:48.150610 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0428 15:37:48.150662 3817 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:48.724107 3817 default_node_network_controller.go:1380] Starting node controller node sync W0428 15:37:48.724107 3817 egressip_healthcheck.go:76] Health checking using insecure connection I0428 15:37:48.724213 3817 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0428 15:37:48.735810 3817 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.68475ms I0428 15:37:48.735939 3817 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-106.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:37:48.735949 3817 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-106.ec2.internal", IP: 10.0.1.106 to PMTUD blocking rules I0428 15:37:48.735969 3817 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-7.ec2.internal", IP: 10.0.2.7 to PMTUD blocking rules I0428 15:37:48.735983 3817 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-7.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0428 15:37:48.747124 3817 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0428 15:37:48.747401 3817 egressservice_node.go:93] Setting up event handlers for Egress Services I0428 15:37:48.747505 3817 egressservice_node.go:183] Starting Egress Services Controller I0428 15:37:48.747524 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0428 15:37:48.747537 3817 shared_informer.go:356] "Caches are synced" controller="egressservices" I0428 15:37:48.747545 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0428 15:37:48.747552 3817 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0428 15:37:48.747579 3817 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0428 15:37:48.747587 3817 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0428 15:37:48.747593 3817 egressservice_node.go:197] Repairing Egress Services I0428 15:37:48.756835 3817 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0428 15:37:48.756845 3817 external_controller.go:278] Starting Admin Policy Based Route Controller I0428 15:37:48.756997 3817 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0428 15:37:48.757009 3817 link_network_manager.go:119] Link manager is running I0428 15:37:48.757041 3817 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0428 15:37:48.757291 3817 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0428 15:37:48.757750 3817 vrf_manager.go:128] VRF manager is running I0428 15:37:48.758014 3817 node_controller_manager.go:437] Removing flows to drop GARP I0428 15:37:48.758023 3817 gateway.go:532] Reconciling gateway with updates I0428 15:37:48.781480 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:48.781493 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:48.781499 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:48.781513 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:48.781512 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:48.781520 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0428 15:37:48.781518 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0428 15:37:48.781525 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:48.781541 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0428 15:37:48.781542 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0428 15:37:48.781503 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0428 15:37:48.781596 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:48.781599 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:48.781602 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0428 15:37:48.781611 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0428 15:37:48.781616 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:48.781625 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0428 15:37:48.781630 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0428 15:37:48.781610 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:48.781630 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:48.781642 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0428 15:37:48.781648 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0428 15:37:48.781649 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:48.781650 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:48.781660 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:48.781660 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:48.781671 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0428 15:37:48.781679 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0428 15:37:48.781692 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0428 15:37:48.781731 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:48.781739 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0428 15:37:48.781737 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:48.781747 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:48.781751 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:48.781755 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0428 15:37:48.781664 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:48.781761 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:48.781763 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0428 15:37:48.781762 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:48.781773 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0428 15:37:48.781766 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:48.781779 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:48.781786 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0428 15:37:48.781789 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:48.781773 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:48.781794 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0428 15:37:48.781803 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:48.781601 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:48.781809 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0428 15:37:48.781812 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0428 15:37:48.781813 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:48.781817 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:48.781513 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0428 15:37:48.781822 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0428 15:37:48.781824 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0428 15:37:48.781828 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0428 15:37:48.781790 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:48.781833 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:48.781757 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0428 15:37:48.781843 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0428 15:37:48.781811 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:48.781857 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0428 15:37:48.781629 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0428 15:37:48.781867 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0428 15:37:48.781763 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0428 15:37:48.781861 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0428 15:37:48.781881 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0428 15:37:48.781891 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0428 15:37:48.781637 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0428 15:37:48.781894 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0428 15:37:48.781908 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0428 15:37:48.781909 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0428 15:37:48.781787 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:48.781924 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0428 15:37:48.781953 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:48.781964 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0428 15:37:48.781648 3817 port_claim.go:71] Svc openshift-ingress/router-default: port 32194 is already open, no action needed I0428 15:37:48.781993 3817 port_claim.go:71] Svc openshift-ingress/router-default: port 31728 is already open, no action needed I0428 15:37:48.781615 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0428 15:37:48.781638 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0428 15:37:48.781601 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0428 15:37:48.781648 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0428 15:37:48.781657 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0428 15:37:48.781659 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0428 15:37:48.781669 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0428 15:37:48.781589 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:48.782132 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0428 15:37:48.781668 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:48.782158 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0428 15:37:48.781742 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0428 15:37:48.781763 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0428 15:37:48.781761 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:48.782184 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0428 15:37:48.781766 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0428 15:37:48.781777 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0428 15:37:48.781791 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0428 15:37:48.782240 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0428 15:37:48.781795 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:48.782247 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0428 15:37:48.782250 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0428 15:37:48.781792 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0428 15:37:48.782256 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0428 15:37:48.781799 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0428 15:37:48.781802 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0428 15:37:48.781803 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:48.782295 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0428 15:37:48.781813 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0428 15:37:48.781815 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0428 15:37:48.781834 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0428 15:37:48.782321 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0428 15:37:48.781847 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0428 15:37:48.782327 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0428 15:37:48.781846 3817 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:48.782334 3817 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:37:48.781851 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0428 15:37:48.781897 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0428 15:37:48.820490 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0428 15:37:48.820519 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0428 15:37:48.820528 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0428 15:37:48.820538 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0428 15:37:48.858552 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0428 15:37:48.858549 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0428 15:37:48.858587 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0428 15:37:48.858607 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0428 15:37:48.858615 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0428 15:37:48.858623 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0428 15:37:48.858629 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0428 15:37:48.858634 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0428 15:37:48.858648 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0428 15:37:48.858664 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0428 15:37:48.858680 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0428 15:37:48.858702 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0428 15:37:48.858712 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0428 15:37:48.858721 3817 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) W0428 15:38:03.228708 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:03.228740 3817 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:03.228745 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:03.231143 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:03.231152 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:03.248667 3817 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:38:03.265764 3817 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:03.414494 3817 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-9.ec2.internal I0428 15:38:03.414528 3817 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-9.ec2.internal: 43.183µs W0428 15:38:03.472782 3817 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-796c59fd79-92mqs IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-796c59fd79-92mqs: no pod IPs found W0428 15:38:03.490854 3817 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:03.631743 3817 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-6d4c95c6c5-bjdcq] creating logical port openshift-ingress_router-default-6d4c95c6c5-bjdcq for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:03.631884 3817 kube.go:256] Updating pod openshift-ingress/router-default-6d4c95c6c5-bjdcq I0428 15:38:03.634773 3817 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-796c59fd79-njh7k] creating logical port openshift-image-registry_image-registry-796c59fd79-njh7k for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:03.634880 3817 kube.go:256] Updating pod openshift-image-registry/image-registry-796c59fd79-njh7k I0428 15:38:03.650459 3817 pod.go:62] [openshift-ingress/router-default-6d4c95c6c5-bjdcq] pod update took 18.602287ms I0428 15:38:03.650489 3817 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-6d4c95c6c5-bjdcq] addLogicalPort annotation time took 18.637646ms I0428 15:38:03.651729 3817 pods.go:271] [openshift-ingress/router-default-6d4c95c6c5-bjdcq] addLogicalPort took 20.002691ms, libovsdb time 924.159µs I0428 15:38:03.656860 3817 pod.go:62] [openshift-image-registry/image-registry-796c59fd79-njh7k] pod update took 21.999525ms I0428 15:38:03.656887 3817 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-796c59fd79-njh7k] addLogicalPort annotation time took 22.026861ms I0428 15:38:03.657974 3817 pods.go:271] [openshift-image-registry/image-registry-796c59fd79-njh7k] addLogicalPort took 23.214995ms, libovsdb time 787.267µs I0428 15:38:03.685729 3817 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-788q5] creating logical port openshift-console_downloads-5c54f4944f-788q5 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:03.685855 3817 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-788q5 I0428 15:38:03.697806 3817 pod.go:62] [openshift-console/downloads-5c54f4944f-788q5] pod update took 11.977582ms I0428 15:38:03.697826 3817 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-788q5] addLogicalPort annotation time took 12.005293ms I0428 15:38:03.698920 3817 pods.go:271] [openshift-console/downloads-5c54f4944f-788q5] addLogicalPort took 13.211513ms, libovsdb time 831.551µs I0428 15:38:03.750524 3817 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-x6zrn] creating logical port openshift-insights_insights-runtime-extractor-x6zrn for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:03.750685 3817 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-x6zrn I0428 15:38:03.763151 3817 pod.go:62] [openshift-insights/insights-runtime-extractor-x6zrn] pod update took 12.49625ms I0428 15:38:03.763172 3817 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-x6zrn] addLogicalPort annotation time took 12.522978ms I0428 15:38:03.764399 3817 pods.go:271] [openshift-insights/insights-runtime-extractor-x6zrn] addLogicalPort took 13.889271ms, libovsdb time 927.578µs I0428 15:38:03.771926 3817 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-nm4nl] creating logical port openshift-dns_dns-default-nm4nl for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:03.772017 3817 kube.go:256] Updating pod openshift-dns/dns-default-nm4nl I0428 15:38:03.788550 3817 pod.go:62] [openshift-dns/dns-default-nm4nl] pod update took 16.563646ms I0428 15:38:03.788584 3817 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-nm4nl] addLogicalPort annotation time took 16.603494ms I0428 15:38:03.789643 3817 pods.go:271] [openshift-dns/dns-default-nm4nl] addLogicalPort took 17.734115ms, libovsdb time 782.797µs I0428 15:38:03.987990 3817 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-106.ec2.internal I0428 15:38:03.988023 3817 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-106.ec2.internal: 38.459µs I0428 15:38:04.028225 3817 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-nmgtr] creating logical port openshift-ingress-canary_ingress-canary-nmgtr for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:04.028342 3817 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-nmgtr I0428 15:38:04.044694 3817 pod.go:62] [openshift-ingress-canary/ingress-canary-nmgtr] pod update took 16.375235ms I0428 15:38:04.044716 3817 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-nmgtr] addLogicalPort annotation time took 16.40213ms I0428 15:38:04.045831 3817 pods.go:271] [openshift-ingress-canary/ingress-canary-nmgtr] addLogicalPort took 17.620862ms, libovsdb time 838.45µs I0428 15:38:04.149688 3817 cni.go:452] [openshift-dns/dns-default-nm4nl 7447d348eac3686e0b02c6b40f60c64dd8b75eb23ca6895056027b1c4e982c1e network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-nm4nl 7447d348eac3686e0b02c6b40f60c64dd8b75eb23ca6895056027b1c4e982c1e network default NAD default NAD key ] I0428 15:38:04.166624 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-nm4nl, hostIfaceName: 7447d348eac3686, network: default, NAD default, SandboxID: "7447d348eac3686e0b02c6b40f60c64dd8b75eb23ca6895056027b1c4e982c1e", PCI device ID: , UID: "5c9c1b2d-ad9c-43ce-868e-9dc77a9194a8", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0428 15:38:04.216836 3817 cni.go:473] [openshift-dns/dns-default-nm4nl 7447d348eac3686e0b02c6b40f60c64dd8b75eb23ca6895056027b1c4e982c1e network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-nm4nl 7447d348eac3686e0b02c6b40f60c64dd8b75eb23ca6895056027b1c4e982c1e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:2f:43:80:36:d5\",\"name\":\"7447d348eac3686\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/873a87d6-8ce2-4892-bba8-210c8e6d208e\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:04.312409 3817 cni.go:452] [openshift-image-registry/image-registry-796c59fd79-njh7k b8effe1735ce702a1b2366b639425185329be116501597d103bf05d90910f88f network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-796c59fd79-njh7k b8effe1735ce702a1b2366b639425185329be116501597d103bf05d90910f88f network default NAD default NAD key ] I0428 15:38:04.321664 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-796c59fd79-njh7k, hostIfaceName: b8effe1735ce702, network: default, NAD default, SandboxID: "b8effe1735ce702a1b2366b639425185329be116501597d103bf05d90910f88f", PCI device ID: , UID: "fb4171a0-f32b-4ee0-bfb2-3d5ede1b2191", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0428 15:38:04.368046 3817 cni.go:473] [openshift-image-registry/image-registry-796c59fd79-njh7k b8effe1735ce702a1b2366b639425185329be116501597d103bf05d90910f88f network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-796c59fd79-njh7k b8effe1735ce702a1b2366b639425185329be116501597d103bf05d90910f88f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:8c:30:f9:81:27\",\"name\":\"b8effe1735ce702\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/23b07cfc-37f2-49e7-bce9-65c73febbc1a\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:38:05.205150 3817 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:05.242033 3817 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:05.306514 3817 cni.go:452] [openshift-ingress-canary/ingress-canary-nmgtr e93bc61b6333b81f072da238592a9885ef6089bac3e0d651958660b6a76eb124 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-nmgtr e93bc61b6333b81f072da238592a9885ef6089bac3e0d651958660b6a76eb124 network default NAD default NAD key ] I0428 15:38:05.316609 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-nmgtr, hostIfaceName: e93bc61b6333b81, network: default, NAD default, SandboxID: "e93bc61b6333b81f072da238592a9885ef6089bac3e0d651958660b6a76eb124", PCI device ID: , UID: "21510bd7-6d59-44af-ad07-5acbde2674b2", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0428 15:38:05.363208 3817 cni.go:473] [openshift-ingress-canary/ingress-canary-nmgtr e93bc61b6333b81f072da238592a9885ef6089bac3e0d651958660b6a76eb124 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-nmgtr e93bc61b6333b81f072da238592a9885ef6089bac3e0d651958660b6a76eb124 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:b9:52:63:86:88\",\"name\":\"e93bc61b6333b81\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1aab966b-ed46-4539-b473-85df1d02e128\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:05.772661 3817 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-tt4dt] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-tt4dt for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:05.772757 3817 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-tt4dt I0428 15:38:05.787258 3817 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-tt4dt] pod update took 14.52947ms I0428 15:38:05.787286 3817 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-tt4dt] addLogicalPort annotation time took 14.556297ms I0428 15:38:05.788423 3817 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-tt4dt] addLogicalPort took 15.781428ms, libovsdb time 783.361µs I0428 15:38:06.022603 3817 cni.go:452] [openshift-console/downloads-5c54f4944f-788q5 0daccc860cf6216b795e7b7fc501aafb39b7add5da91689273b8e11a3ce73ff3 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-788q5 0daccc860cf6216b795e7b7fc501aafb39b7add5da91689273b8e11a3ce73ff3 network default NAD default NAD key ] I0428 15:38:06.026373 3817 cni.go:452] [openshift-insights/insights-runtime-extractor-x6zrn f1c43a8dcc471fc78028362c06efb24dfa22d89b8ab49c67b80615beb38892c7 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-x6zrn f1c43a8dcc471fc78028362c06efb24dfa22d89b8ab49c67b80615beb38892c7 network default NAD default NAD key ] I0428 15:38:06.027443 3817 cni.go:452] [openshift-ingress/router-default-6d4c95c6c5-bjdcq d749b50b5a3dd31d51a6b5aed61776bfae0c47aea429bfd1dc47e4fd06ccfd7f network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-6d4c95c6c5-bjdcq d749b50b5a3dd31d51a6b5aed61776bfae0c47aea429bfd1dc47e4fd06ccfd7f network default NAD default NAD key ] I0428 15:38:06.034331 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-788q5, hostIfaceName: 0daccc860cf6216, network: default, NAD default, SandboxID: "0daccc860cf6216b795e7b7fc501aafb39b7add5da91689273b8e11a3ce73ff3", PCI device ID: , UID: "dba15ec7-6abc-435e-9daa-571fd0397bd1", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0428 15:38:06.039070 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-6d4c95c6c5-bjdcq, hostIfaceName: d749b50b5a3dd31, network: default, NAD default, SandboxID: "d749b50b5a3dd31d51a6b5aed61776bfae0c47aea429bfd1dc47e4fd06ccfd7f", PCI device ID: , UID: "13f9e8f7-932b-4013-aca0-7d04766d66f5", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0428 15:38:06.051551 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-x6zrn, hostIfaceName: f1c43a8dcc471fc, network: default, NAD default, SandboxID: "f1c43a8dcc471fc78028362c06efb24dfa22d89b8ab49c67b80615beb38892c7", PCI device ID: , UID: "9031afdd-4219-455d-8e9e-eb76ffdbb00f", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0428 15:38:06.081773 3817 cni.go:473] [openshift-console/downloads-5c54f4944f-788q5 0daccc860cf6216b795e7b7fc501aafb39b7add5da91689273b8e11a3ce73ff3 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-788q5 0daccc860cf6216b795e7b7fc501aafb39b7add5da91689273b8e11a3ce73ff3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:86:b3:16:12:02\",\"name\":\"0daccc860cf6216\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/54feda30-5a99-4e6c-98f8-195f1a78156e\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:06.158880 3817 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-tt4dt f9dab53c55bbe195863fc4ac32e4b1486dee8e3a7f0093efc688406a925428c1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-tt4dt f9dab53c55bbe195863fc4ac32e4b1486dee8e3a7f0093efc688406a925428c1 network default NAD default NAD key ] I0428 15:38:06.169261 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-tt4dt, hostIfaceName: f9dab53c55bbe19, network: default, NAD default, SandboxID: "f9dab53c55bbe195863fc4ac32e4b1486dee8e3a7f0093efc688406a925428c1", PCI device ID: , UID: "ebdfc8d1-1ff8-4e27-b64e-1750aa4a4262", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0428 15:38:06.212580 3817 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-tt4dt f9dab53c55bbe195863fc4ac32e4b1486dee8e3a7f0093efc688406a925428c1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-tt4dt f9dab53c55bbe195863fc4ac32e4b1486dee8e3a7f0093efc688406a925428c1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:ed:a8:1e:b5:54\",\"name\":\"f9dab53c55bbe19\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9cbf0b81-227d-477b-b583-5727301ec61a\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:06.303405 3817 cni.go:473] [openshift-insights/insights-runtime-extractor-x6zrn f1c43a8dcc471fc78028362c06efb24dfa22d89b8ab49c67b80615beb38892c7 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-x6zrn f1c43a8dcc471fc78028362c06efb24dfa22d89b8ab49c67b80615beb38892c7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:3e:29:ff:f1:1f\",\"name\":\"f1c43a8dcc471fc\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/50911e65-f118-48c3-98ce-6d20981c9d76\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:06.303481 3817 cni.go:473] [openshift-ingress/router-default-6d4c95c6c5-bjdcq d749b50b5a3dd31d51a6b5aed61776bfae0c47aea429bfd1dc47e4fd06ccfd7f network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-6d4c95c6c5-bjdcq d749b50b5a3dd31d51a6b5aed61776bfae0c47aea429bfd1dc47e4fd06ccfd7f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:1d:54:97:45:4a\",\"name\":\"d749b50b5a3dd31\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5327610f-6a42-4914-b091-95d39ec25abd\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:38:07.249026 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:07.249051 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:08.249144 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:08.249168 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:08.249175 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default W0428 15:38:08.254996 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0428 15:38:08.255017 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0428 15:38:08.384571 3817 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0428 15:38:09.266134 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0428 15:38:10.226475 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0428 15:38:10.226504 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0428 15:38:10.259765 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0428 15:38:11.970449 3817 base_network_controller_pods.go:487] [default/openshift-console/console-957cdcb8f-6cq6s] creating logical port openshift-console_console-957cdcb8f-6cq6s for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:11.970581 3817 kube.go:256] Updating pod openshift-console/console-957cdcb8f-6cq6s I0428 15:38:11.983666 3817 pod.go:62] [openshift-console/console-957cdcb8f-6cq6s] pod update took 13.137461ms I0428 15:38:11.983690 3817 base_network_controller_pods.go:951] [default/openshift-console/console-957cdcb8f-6cq6s] addLogicalPort annotation time took 13.167112ms I0428 15:38:11.984753 3817 pods.go:271] [openshift-console/console-957cdcb8f-6cq6s] addLogicalPort took 14.328089ms, libovsdb time 769.643µs I0428 15:38:12.385143 3817 cni.go:452] [openshift-console/console-957cdcb8f-6cq6s 430554254cbac2c3dd7b77cfc4977b62be3f3d2f4f42d58067386de68395f931 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-957cdcb8f-6cq6s 430554254cbac2c3dd7b77cfc4977b62be3f3d2f4f42d58067386de68395f931 network default NAD default NAD key ] I0428 15:38:12.396906 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-957cdcb8f-6cq6s, hostIfaceName: 430554254cbac2c, network: default, NAD default, SandboxID: "430554254cbac2c3dd7b77cfc4977b62be3f3d2f4f42d58067386de68395f931", PCI device ID: , UID: "ae33fe0c-d202-4562-8011-af060372b505", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0428 15:38:12.448140 3817 cni.go:473] [openshift-console/console-957cdcb8f-6cq6s 430554254cbac2c3dd7b77cfc4977b62be3f3d2f4f42d58067386de68395f931 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-957cdcb8f-6cq6s 430554254cbac2c3dd7b77cfc4977b62be3f3d2f4f42d58067386de68395f931 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:db:22:d1:74:91\",\"name\":\"430554254cbac2c\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3bbc328a-504c-42d9-9fe9-07e051238151\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:38:13.478265 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:13.478293 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:16.280495 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 15:38:16.291362 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0428 15:38:17.252542 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:17.252591 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0428 15:38:17.252600 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0428 15:38:18.509889 3817 base_network_controller_pods.go:487] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-8x5t7] creating logical port openshift-monitoring_kube-state-metrics-6dfffbb599-8x5t7 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:18.510000 3817 kube.go:256] Updating pod openshift-monitoring/kube-state-metrics-6dfffbb599-8x5t7 I0428 15:38:18.538125 3817 pod.go:62] [openshift-monitoring/kube-state-metrics-6dfffbb599-8x5t7] pod update took 28.163994ms I0428 15:38:18.538149 3817 base_network_controller_pods.go:951] [default/openshift-monitoring/kube-state-metrics-6dfffbb599-8x5t7] addLogicalPort annotation time took 28.194526ms I0428 15:38:18.539436 3817 pods.go:271] [openshift-monitoring/kube-state-metrics-6dfffbb599-8x5t7] addLogicalPort took 29.573868ms, libovsdb time 972.99µs I0428 15:38:19.495595 3817 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:19.495738 3817 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0428 15:38:19.529639 3817 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 33.935642ms I0428 15:38:19.529667 3817 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 33.970375ms I0428 15:38:19.530943 3817 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 35.386167ms, libovsdb time 923.557µs I0428 15:38:20.522112 3817 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-5f7b998cbb-5dxrt] creating logical port openshift-monitoring_thanos-querier-5f7b998cbb-5dxrt for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:20.522243 3817 kube.go:256] Updating pod openshift-monitoring/thanos-querier-5f7b998cbb-5dxrt I0428 15:38:20.541596 3817 pod.go:62] [openshift-monitoring/thanos-querier-5f7b998cbb-5dxrt] pod update took 19.378275ms I0428 15:38:20.541627 3817 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-5f7b998cbb-5dxrt] addLogicalPort annotation time took 19.417542ms I0428 15:38:20.543229 3817 pods.go:271] [openshift-monitoring/thanos-querier-5f7b998cbb-5dxrt] addLogicalPort took 21.139081ms, libovsdb time 1.176447ms W0428 15:38:22.341041 3817 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:22.341065 3817 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:22.341072 3817 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 15:38:23.062463 3817 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-7d968657fc-t4w5k] creating logical port openshift-monitoring_metrics-server-7d968657fc-t4w5k for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:23.062603 3817 kube.go:256] Updating pod openshift-monitoring/metrics-server-7d968657fc-t4w5k I0428 15:38:23.078437 3817 pod.go:62] [openshift-monitoring/metrics-server-7d968657fc-t4w5k] pod update took 15.853423ms I0428 15:38:23.078466 3817 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-7d968657fc-t4w5k] addLogicalPort annotation time took 15.905486ms I0428 15:38:23.079742 3817 pods.go:271] [openshift-monitoring/metrics-server-7d968657fc-t4w5k] addLogicalPort took 17.296859ms, libovsdb time 945.229µs I0428 15:38:23.135920 3817 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-zsgc5] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-zsgc5 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:23.136026 3817 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-zsgc5 I0428 15:38:23.147951 3817 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-zsgc5] pod update took 11.956157ms I0428 15:38:23.147980 3817 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-zsgc5] addLogicalPort annotation time took 11.989594ms I0428 15:38:23.149211 3817 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-zsgc5] addLogicalPort took 13.310155ms, libovsdb time 922.407µs W0428 15:38:23.316936 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 15:38:23.946603 3817 cni.go:452] [openshift-monitoring/kube-state-metrics-6dfffbb599-8x5t7 f80ef596df950e03274eed21d8b56f0214698998f685da57b7bd7a1a9abcfc74 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-8x5t7 f80ef596df950e03274eed21d8b56f0214698998f685da57b7bd7a1a9abcfc74 network default NAD default NAD key ] I0428 15:38:23.947545 3817 cni.go:452] [openshift-monitoring/thanos-querier-5f7b998cbb-5dxrt a502fa2cb5abdddbaf8fb7b9970164510255aa2a4132a7aaa06a9cc1be7d7f56 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-5f7b998cbb-5dxrt a502fa2cb5abdddbaf8fb7b9970164510255aa2a4132a7aaa06a9cc1be7d7f56 network default NAD default NAD key ] I0428 15:38:23.948901 3817 cni.go:452] [openshift-multus/network-metrics-daemon-c85fc 61b84403f4b38dbdabe51bd42d3f003f3e4ed906d1272ed38a55213bc407f90f network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-c85fc 61b84403f4b38dbdabe51bd42d3f003f3e4ed906d1272ed38a55213bc407f90f network default NAD default NAD key ] I0428 15:38:23.951123 3817 cni.go:452] [openshift-monitoring/alertmanager-main-0 2b7e67fc815ed7e75f22e5ced784369fe41d957aeb5817b7c2d06c1c4740b82b network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 2b7e67fc815ed7e75f22e5ced784369fe41d957aeb5817b7c2d06c1c4740b82b network default NAD default NAD key ] I0428 15:38:23.951131 3817 cni.go:452] [openshift-network-diagnostics/network-check-target-gzrrf e8f5717b86bd4f916712130cbdfe8f3781c0994ecdb0d161c00d0a0cd61438bd network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-gzrrf e8f5717b86bd4f916712130cbdfe8f3781c0994ecdb0d161c00d0a0cd61438bd network default NAD default NAD key ] I0428 15:38:23.952602 3817 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-zsgc5 ce1a10fda081b556af176905bbfb8b0e6b929dc1d6b55cc759a2f83c59fd326c network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-zsgc5 ce1a10fda081b556af176905bbfb8b0e6b929dc1d6b55cc759a2f83c59fd326c network default NAD default NAD key ] I0428 15:38:23.952622 3817 cni.go:452] [openshift-monitoring/metrics-server-7d968657fc-t4w5k ea80ef66dbcf44c20a54370d411ae678f39b1f0cf3b82cd15ae18c1ca9282d05 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-7d968657fc-t4w5k ea80ef66dbcf44c20a54370d411ae678f39b1f0cf3b82cd15ae18c1ca9282d05 network default NAD default NAD key ] I0428 15:38:23.966116 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 2b7e67fc815ed7e, network: default, NAD default, SandboxID: "2b7e67fc815ed7e75f22e5ced784369fe41d957aeb5817b7c2d06c1c4740b82b", PCI device ID: , UID: "ff47b57f-c99c-4829-a6c2-9b1c3ce2382c", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0428 15:38:23.977330 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: kube-state-metrics-6dfffbb599-8x5t7, hostIfaceName: f80ef596df950e0, network: default, NAD default, SandboxID: "f80ef596df950e03274eed21d8b56f0214698998f685da57b7bd7a1a9abcfc74", PCI device ID: , UID: "73a2f107-004d-4d00-9542-4b232ebbabb6", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0428 15:38:23.977443 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-zsgc5, hostIfaceName: ce1a10fda081b55, network: default, NAD default, SandboxID: "ce1a10fda081b556af176905bbfb8b0e6b929dc1d6b55cc759a2f83c59fd326c", PCI device ID: , UID: "1ad9f46e-1ac7-45c3-9e85-30e085447db8", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0428 15:38:23.978192 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-c85fc, hostIfaceName: 61b84403f4b38db, network: default, NAD default, SandboxID: "61b84403f4b38dbdabe51bd42d3f003f3e4ed906d1272ed38a55213bc407f90f", PCI device ID: , UID: "fd81b0cb-ebef-4e91-999f-3b7f8ce210fe", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0428 15:38:23.978884 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-7d968657fc-t4w5k, hostIfaceName: ea80ef66dbcf44c, network: default, NAD default, SandboxID: "ea80ef66dbcf44c20a54370d411ae678f39b1f0cf3b82cd15ae18c1ca9282d05", PCI device ID: , UID: "a26194d2-017c-4adc-bbc8-7bfa066db887", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0428 15:38:23.979210 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-5f7b998cbb-5dxrt, hostIfaceName: a502fa2cb5abddd, network: default, NAD default, SandboxID: "a502fa2cb5abdddbaf8fb7b9970164510255aa2a4132a7aaa06a9cc1be7d7f56", PCI device ID: , UID: "b86c3614-e34b-47d3-bbe8-ce259f139d00", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0428 15:38:23.984715 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-gzrrf, hostIfaceName: e8f5717b86bd4f9, network: default, NAD default, SandboxID: "e8f5717b86bd4f916712130cbdfe8f3781c0994ecdb0d161c00d0a0cd61438bd", PCI device ID: , UID: "66f5e84b-3fa8-4f7d-9f49-e124cf69396d", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0428 15:38:24.041684 3817 cni.go:473] [openshift-monitoring/alertmanager-main-0 2b7e67fc815ed7e75f22e5ced784369fe41d957aeb5817b7c2d06c1c4740b82b network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 2b7e67fc815ed7e75f22e5ced784369fe41d957aeb5817b7c2d06c1c4740b82b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:c6:49:86:f1:93\",\"name\":\"2b7e67fc815ed7e\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6f0859d1-f6d2-4e05-b629-692e7512a718\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:24.131273 3817 cni.go:473] [openshift-monitoring/kube-state-metrics-6dfffbb599-8x5t7 f80ef596df950e03274eed21d8b56f0214698998f685da57b7bd7a1a9abcfc74 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/kube-state-metrics-6dfffbb599-8x5t7 f80ef596df950e03274eed21d8b56f0214698998f685da57b7bd7a1a9abcfc74 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:da:bc:87:f0:36\",\"name\":\"f80ef596df950e0\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8a97f7c6-a160-4da3-91e5-6f01ee213d86\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:24.261318 3817 cni.go:473] [openshift-monitoring/metrics-server-7d968657fc-t4w5k ea80ef66dbcf44c20a54370d411ae678f39b1f0cf3b82cd15ae18c1ca9282d05 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-7d968657fc-t4w5k ea80ef66dbcf44c20a54370d411ae678f39b1f0cf3b82cd15ae18c1ca9282d05 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:83:c0:47:5a:62\",\"name\":\"ea80ef66dbcf44c\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/84179cdf-37fe-40c9-9732-5604fe5c703c\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:24.261439 3817 cni.go:473] [openshift-network-diagnostics/network-check-target-gzrrf e8f5717b86bd4f916712130cbdfe8f3781c0994ecdb0d161c00d0a0cd61438bd network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-gzrrf e8f5717b86bd4f916712130cbdfe8f3781c0994ecdb0d161c00d0a0cd61438bd network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:61:8c:0b:c1:0a\",\"name\":\"e8f5717b86bd4f9\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/758b34e5-a6b5-47a6-a919-eaacf7738eec\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:24.261447 3817 cni.go:473] [openshift-multus/network-metrics-daemon-c85fc 61b84403f4b38dbdabe51bd42d3f003f3e4ed906d1272ed38a55213bc407f90f network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-c85fc 61b84403f4b38dbdabe51bd42d3f003f3e4ed906d1272ed38a55213bc407f90f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:9e:40:43:7a:47\",\"name\":\"61b84403f4b38db\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a959625e-cfcc-4ee9-bb31-853cad0a4e80\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:24.261909 3817 cni.go:473] [openshift-monitoring/thanos-querier-5f7b998cbb-5dxrt a502fa2cb5abdddbaf8fb7b9970164510255aa2a4132a7aaa06a9cc1be7d7f56 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-5f7b998cbb-5dxrt a502fa2cb5abdddbaf8fb7b9970164510255aa2a4132a7aaa06a9cc1be7d7f56 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:55:a5:bd:50:28\",\"name\":\"a502fa2cb5abddd\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fee40091-e18d-4156-83a9-dfc58cd9acaa\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:38:24.262003 3817 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-zsgc5 ce1a10fda081b556af176905bbfb8b0e6b929dc1d6b55cc759a2f83c59fd326c network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-zsgc5 ce1a10fda081b556af176905bbfb8b0e6b929dc1d6b55cc759a2f83c59fd326c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"52:54:92:a6:71:9f\",\"name\":\"ce1a10fda081b55\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/46729408-bfb3-4162-83f8-1aa64a40938a\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:38:24.320617 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads I0428 15:38:24.917185 3817 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:38:24.917368 3817 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0428 15:38:24.941241 3817 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 23.931535ms I0428 15:38:24.941268 3817 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 23.965601ms I0428 15:38:24.942700 3817 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 25.536007ms, libovsdb time 990.269µs W0428 15:38:25.339976 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service openshift-console/downloads W0428 15:38:25.774714 3817 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-56b489d6fc-lccqq IP address from the namespace address-set, err: pod openshift-console/console-56b489d6fc-lccqq: no pod IPs found W0428 15:38:26.205951 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:26.205984 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0428 15:38:26.252798 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0428 15:38:27.285412 3817 cni.go:452] [openshift-monitoring/prometheus-k8s-1 12f17c134e226c04cf5e60e10db29518873673636a9d05b62f52bcd21049605d network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 12f17c134e226c04cf5e60e10db29518873673636a9d05b62f52bcd21049605d network default NAD default NAD key ] I0428 15:38:27.297831 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 12f17c134e226c0, network: default, NAD default, SandboxID: "12f17c134e226c04cf5e60e10db29518873673636a9d05b62f52bcd21049605d", PCI device ID: , UID: "33d9ab42-6cc7-400b-90ce-49594681d119", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0428 15:38:27.354703 3817 cni.go:473] [openshift-monitoring/prometheus-k8s-1 12f17c134e226c04cf5e60e10db29518873673636a9d05b62f52bcd21049605d network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 12f17c134e226c04cf5e60e10db29518873673636a9d05b62f52bcd21049605d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:2c:18:1f:fd:45\",\"name\":\"12f17c134e226c0\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/03500bae-742b-454e-b707-17f5e2f0d150\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:38:28.355281 3817 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:28.355306 3817 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:28.355321 3817 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0428 15:38:28.759478 3817 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:28.760109 3817 obj_retry.go:498] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:29.118457 3817 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:29.118546 3817 obj_retry.go:492] Detected object openshift-image-registry/image-registry-7fc7c49d95-dgxkp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:29.131403 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-7fc7c49d95-dgxkp I0428 15:38:29.131435 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-7fc7c49d95-dgxkp W0428 15:38:29.341925 3817 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.341945 3817 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.437087 3817 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:38:29.512772 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:38:29.603167 3817 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.603190 3817 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:38:29.688167 3817 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-69f694b5b7-j8xh7 IP address from the namespace address-set, err: pod openshift-console/console-69f694b5b7-j8xh7: no pod IPs found W0428 15:38:29.690861 3817 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-69f694b5b7-9ddmp IP address from the namespace address-set, err: pod openshift-console/console-69f694b5b7-9ddmp: no pod IPs found W0428 15:38:30.520649 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0428 15:38:31.423080 3817 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.423726 3817 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.423759 3817 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.423768 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0428 15:38:31.423781 3817 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:38:31.423786 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... W0428 15:38:32.350398 3817 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 15:38:32.449584 3817 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:32.449621 3817 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:32.449642 3817 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:32.449656 3817 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dggfm8 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:38:32.989201 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:38:32.989227 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0428 15:38:34.772466 3817 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0428 15:38:34.787798 3817 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0428 15:38:37.415688 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 15:38:37.415708 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 15:38:37.415713 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0428 15:38:37.415718 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier I0428 15:38:41.510255 3817 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-n8rgx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:41.510821 3817 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-n8rgx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:42.411270 3817 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-n8rgx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:42.411292 3817 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-n8rgx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:42.414030 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-957cdcb8f-n8rgx I0428 15:38:42.414055 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-957cdcb8f-n8rgx W0428 15:38:46.193865 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0428 15:38:49.997234 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:49.997261 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:49.997269 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:49.997288 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:49.997295 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:49.997303 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:50.350649 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:38:50.350671 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:38:50.386246 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0428 15:38:51.235807 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics I0428 15:38:51.495301 3817 cni.go:452] [openshift-image-registry/image-registry-796c59fd79-njh7k b8effe1735ce702a1b2366b639425185329be116501597d103bf05d90910f88f network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-796c59fd79-njh7k b8effe1735ce702a1b2366b639425185329be116501597d103bf05d90910f88f network default NAD default NAD key ] I0428 15:38:51.573432 3817 cni.go:473] [openshift-image-registry/image-registry-796c59fd79-njh7k b8effe1735ce702a1b2366b639425185329be116501597d103bf05d90910f88f network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-796c59fd79-njh7k b8effe1735ce702a1b2366b639425185329be116501597d103bf05d90910f88f network default NAD default NAD key default], result "{}", err I0428 15:38:51.646981 3817 obj_retry.go:498] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:51.647021 3817 pods.go:176] Deleting pod: openshift-image-registry/image-registry-796c59fd79-njh7k I0428 15:38:51.648377 3817 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-796c59fd79-njh7k, ips: 10.129.0.6 I0428 15:38:51.648404 3817 obj_retry.go:498] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:52.427303 3817 obj_retry.go:498] Detected object openshift-console/console-56b489d6fc-b8dxl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:52.427886 3817 obj_retry.go:498] Detected object openshift-console/console-56b489d6fc-b8dxl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:38:52.483432 3817 obj_retry.go:492] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:52.483452 3817 obj_retry.go:492] Detected object openshift-image-registry/image-registry-796c59fd79-njh7k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:52.488893 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-796c59fd79-njh7k I0428 15:38:52.488914 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-796c59fd79-njh7k I0428 15:38:53.234344 3817 obj_retry.go:492] Detected object openshift-console/console-56b489d6fc-b8dxl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:53.234369 3817 obj_retry.go:492] Detected object openshift-console/console-56b489d6fc-b8dxl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:38:53.238991 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-56b489d6fc-b8dxl I0428 15:38:53.239017 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-56b489d6fc-b8dxl W0428 15:38:59.925106 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:59.925125 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:38:59.925130 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:39:00.381907 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0428 15:39:16.320146 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0428 15:39:17.682957 3817 cni.go:452] [openshift-console/console-957cdcb8f-6cq6s 430554254cbac2c3dd7b77cfc4977b62be3f3d2f4f42d58067386de68395f931 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-957cdcb8f-6cq6s 430554254cbac2c3dd7b77cfc4977b62be3f3d2f4f42d58067386de68395f931 network default NAD default NAD key ] I0428 15:39:17.765622 3817 cni.go:473] [openshift-console/console-957cdcb8f-6cq6s 430554254cbac2c3dd7b77cfc4977b62be3f3d2f4f42d58067386de68395f931 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-957cdcb8f-6cq6s 430554254cbac2c3dd7b77cfc4977b62be3f3d2f4f42d58067386de68395f931 network default NAD default NAD key default], result "{}", err I0428 15:39:17.834934 3817 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-6cq6s of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:17.834972 3817 pods.go:176] Deleting pod: openshift-console/console-957cdcb8f-6cq6s I0428 15:39:17.837736 3817 pods.go:236] Attempting to release IPs for pod: openshift-console/console-957cdcb8f-6cq6s, ips: 10.129.0.12 I0428 15:39:17.837829 3817 obj_retry.go:498] Detected object openshift-console/console-957cdcb8f-6cq6s of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:18.576369 3817 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-6cq6s of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:18.576400 3817 obj_retry.go:492] Detected object openshift-console/console-957cdcb8f-6cq6s of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:18.581468 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-957cdcb8f-6cq6s I0428 15:39:18.581498 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-957cdcb8f-6cq6s I0428 15:39:19.357521 3817 obj_retry.go:498] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:19.358098 3817 obj_retry.go:498] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:20.328350 3817 obj_retry.go:492] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:20.328380 3817 obj_retry.go:492] Detected object openshift-ingress/router-default-5f578f8577-xnnpr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:20.338374 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-5f578f8577-xnnpr I0428 15:39:20.338403 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-5f578f8577-xnnpr W0428 15:39:25.623259 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:39:25.623283 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:39:25.623293 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:39:25.623298 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:39:25.652124 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:39:25.652143 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0428 15:39:33.165804 3817 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0428 15:39:41.095966 3817 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:41.096519 3817 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:41.662996 3817 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:41.663020 3817 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:41.669729 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0428 15:39:41.669754 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0428 15:39:44.753271 3817 cni.go:452] [openshift-monitoring/prometheus-k8s-1 12f17c134e226c04cf5e60e10db29518873673636a9d05b62f52bcd21049605d network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 12f17c134e226c04cf5e60e10db29518873673636a9d05b62f52bcd21049605d network default NAD default NAD key ] I0428 15:39:44.840346 3817 cni.go:473] [openshift-monitoring/prometheus-k8s-1 12f17c134e226c04cf5e60e10db29518873673636a9d05b62f52bcd21049605d network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 12f17c134e226c04cf5e60e10db29518873673636a9d05b62f52bcd21049605d network default NAD default NAD key default], result "{}", err I0428 15:39:44.913992 3817 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:44.914021 3817 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0428 15:39:44.915205 3817 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.129.0.18 I0428 15:39:44.915235 3817 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:39:45.197801 3817 base_network_controller_pods.go:487] [default/openshift-console/console-7ff7689755-phs5w] creating logical port openshift-console_console-7ff7689755-phs5w for pod on switch ip-10-0-0-9.ec2.internal I0428 15:39:45.197917 3817 kube.go:256] Updating pod openshift-console/console-7ff7689755-phs5w I0428 15:39:45.221835 3817 pod.go:62] [openshift-console/console-7ff7689755-phs5w] pod update took 23.954401ms I0428 15:39:45.221859 3817 base_network_controller_pods.go:951] [default/openshift-console/console-7ff7689755-phs5w] addLogicalPort annotation time took 23.981912ms I0428 15:39:45.223152 3817 pods.go:271] [openshift-console/console-7ff7689755-phs5w] addLogicalPort took 25.364664ms, libovsdb time 930.217µs I0428 15:39:45.566188 3817 cni.go:452] [openshift-console/console-7ff7689755-phs5w c887441d111301bd423b4117a1f37165b3f77b02519800343a18b6cd460bf439 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-7ff7689755-phs5w c887441d111301bd423b4117a1f37165b3f77b02519800343a18b6cd460bf439 network default NAD default NAD key ] I0428 15:39:45.578706 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-7ff7689755-phs5w, hostIfaceName: c887441d111301b, network: default, NAD default, SandboxID: "c887441d111301bd423b4117a1f37165b3f77b02519800343a18b6cd460bf439", PCI device ID: , UID: "b3a00ec5-8cc4-4284-882e-9b370f4fbf8e", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0428 15:39:45.630939 3817 cni.go:473] [openshift-console/console-7ff7689755-phs5w c887441d111301bd423b4117a1f37165b3f77b02519800343a18b6cd460bf439 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-7ff7689755-phs5w c887441d111301bd423b4117a1f37165b3f77b02519800343a18b6cd460bf439 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:1e:dc:ea:9c:e0\",\"name\":\"c887441d111301b\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/96b30700-f2b2-4076-8528-2023f7272e39\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:39:45.686809 3817 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:45.686902 3817 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:39:45.693084 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0428 15:39:45.693108 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0428 15:39:45.738431 3817 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:39:45.738578 3817 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0428 15:39:45.763206 3817 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 24.681262ms I0428 15:39:45.763236 3817 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 24.715539ms I0428 15:39:45.764502 3817 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 26.089688ms, libovsdb time 950.682µs I0428 15:39:46.099360 3817 cni.go:452] [openshift-monitoring/prometheus-k8s-1 e619ad31434ce57156b4613a7dfb06fc67f644439ea9e1391f0b29bcab84cbf8 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 e619ad31434ce57156b4613a7dfb06fc67f644439ea9e1391f0b29bcab84cbf8 network default NAD default NAD key ] I0428 15:39:46.110223 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: e619ad31434ce57, network: default, NAD default, SandboxID: "e619ad31434ce57156b4613a7dfb06fc67f644439ea9e1391f0b29bcab84cbf8", PCI device ID: , UID: "9bc0445c-cfa4-4ce8-ba2a-25f5cc67c5d1", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0428 15:39:46.155740 3817 cni.go:473] [openshift-monitoring/prometheus-k8s-1 e619ad31434ce57156b4613a7dfb06fc67f644439ea9e1391f0b29bcab84cbf8 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 e619ad31434ce57156b4613a7dfb06fc67f644439ea9e1391f0b29bcab84cbf8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:38:a3:18:78:e0\",\"name\":\"e619ad31434ce57\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/15248cc1-5c20-422d-8fcd-d40d6220acc8\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:39:55.750153 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0428 15:40:03.310461 3817 cni.go:452] [openshift-monitoring/alertmanager-main-0 2b7e67fc815ed7e75f22e5ced784369fe41d957aeb5817b7c2d06c1c4740b82b network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 2b7e67fc815ed7e75f22e5ced784369fe41d957aeb5817b7c2d06c1c4740b82b network default NAD default NAD key ] I0428 15:40:03.382605 3817 cni.go:473] [openshift-monitoring/alertmanager-main-0 2b7e67fc815ed7e75f22e5ced784369fe41d957aeb5817b7c2d06c1c4740b82b network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 2b7e67fc815ed7e75f22e5ced784369fe41d957aeb5817b7c2d06c1c4740b82b network default NAD default NAD key default], result "{}", err I0428 15:40:03.444919 3817 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:03.444959 3817 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0428 15:40:03.446411 3817 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.129.0.14 I0428 15:40:03.446443 3817 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:03.778735 3817 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:03.778765 3817 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:03.785291 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0428 15:40:03.785341 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0428 15:40:03.821087 3817 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:40:03.821217 3817 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0428 15:40:03.841465 3817 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 20.295002ms I0428 15:40:03.841493 3817 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 20.328239ms I0428 15:40:03.843001 3817 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 21.932226ms, libovsdb time 1.154719ms I0428 15:40:04.182602 3817 cni.go:452] [openshift-monitoring/alertmanager-main-0 484bb22942743f916cab87a3313659ad9fe88b3ff8ae5b068885e6a5bf09b42f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 484bb22942743f916cab87a3313659ad9fe88b3ff8ae5b068885e6a5bf09b42f network default NAD default NAD key ] I0428 15:40:04.191853 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 484bb22942743f9, network: default, NAD default, SandboxID: "484bb22942743f916cab87a3313659ad9fe88b3ff8ae5b068885e6a5bf09b42f", PCI device ID: , UID: "bffdedc6-eb5c-4ec0-9c4a-7b3e881f4939", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0428 15:40:04.242252 3817 cni.go:473] [openshift-monitoring/alertmanager-main-0 484bb22942743f916cab87a3313659ad9fe88b3ff8ae5b068885e6a5bf09b42f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 484bb22942743f916cab87a3313659ad9fe88b3ff8ae5b068885e6a5bf09b42f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:88:34:30:d5:f7\",\"name\":\"484bb22942743f9\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/22ceb6ea-8e3b-4254-9705-7606b15814d1\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:40:05.661402 3817 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:05.662037 3817 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:06.547404 3817 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:06.547433 3817 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623180-zgjmh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:10.501097 3817 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-zgjjj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:10.501745 3817 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-zgjjj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:10.792328 3817 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-zgjjj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:10.792356 3817 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-zgjjj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:10.798754 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5d45f7999c-zgjjj I0428 15:40:10.798782 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5d45f7999c-zgjjj I0428 15:40:21.084660 3817 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-wmx4k of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:21.085234 3817 obj_retry.go:498] Detected object openshift-console/console-5d45f7999c-wmx4k of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:21.586288 3817 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-wmx4k of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:21.586314 3817 obj_retry.go:492] Detected object openshift-console/console-5d45f7999c-wmx4k of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:21.592658 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-5d45f7999c-wmx4k I0428 15:40:21.592689 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-5d45f7999c-wmx4k W0428 15:40:24.233965 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:40:24.233988 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:40:24.233996 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0428 15:40:47.001312 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0428 15:40:47.001333 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0428 15:40:48.292310 3817 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:48.292853 3817 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:40:48.951542 3817 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:48.951584 3817 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:40:48.958593 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0428 15:40:48.958620 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0428 15:42:53.375198 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=10 I0428 15:43:18.477250 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=5 W0428 15:43:39.338338 3817 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0428 15:43:50.391910 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0428 15:43:50.391999 3817 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-8j7mw IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-8j7mw: no pod IPs found I0428 15:44:00.460719 3817 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0428 15:44:56.498341 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=8 I0428 15:45:28.385489 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=130 I0428 15:45:33.376806 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=8 I0428 15:45:37.461278 3817 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=8 I0428 15:45:47.761076 3817 namespace.go:98] [tsf] adding namespace I0428 15:45:47.761975 3817 namespace.go:102] [tsf] adding namespace took 868.132µs I0428 15:45:47.776401 3817 namespace.go:142] [tsf] updating namespace I0428 15:45:47.791904 3817 namespace.go:142] [tsf] updating namespace I0428 15:45:51.427924 3817 namespace.go:98] [cert-manager-operator] adding namespace I0428 15:45:51.428781 3817 namespace.go:102] [cert-manager-operator] adding namespace took 833.177µs I0428 15:45:51.435620 3817 namespace.go:98] [konflux-ui] adding namespace I0428 15:45:51.436298 3817 namespace.go:102] [konflux-ui] adding namespace took 655.084µs I0428 15:45:51.437607 3817 namespace.go:98] [rhtpa-operator] adding namespace I0428 15:45:51.438140 3817 namespace.go:102] [rhtpa-operator] adding namespace took 519.185µs I0428 15:45:51.438960 3817 namespace.go:98] [tsf-tpa] adding namespace I0428 15:45:51.439608 3817 namespace.go:102] [tsf-tpa] adding namespace took 634.391µs I0428 15:45:51.450583 3817 namespace.go:98] [rhbk-operator] adding namespace I0428 15:45:51.450636 3817 namespace.go:98] [tsf-keycloak] adding namespace I0428 15:45:51.450726 3817 namespace.go:98] [tsf-tas] adding namespace I0428 15:45:51.450762 3817 namespace.go:98] [konflux-operator] adding namespace I0428 15:45:51.451267 3817 namespace.go:102] [rhbk-operator] adding namespace took 669.141µs I0428 15:45:51.451875 3817 namespace.go:102] [tsf-keycloak] adding namespace took 1.220687ms I0428 15:45:51.452401 3817 namespace.go:102] [tsf-tas] adding namespace took 1.656982ms I0428 15:45:51.452903 3817 namespace.go:102] [konflux-operator] adding namespace took 2.134708ms I0428 15:45:51.464453 3817 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:45:51.493729 3817 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:45:51.507226 3817 namespace.go:142] [konflux-ui] updating namespace I0428 15:45:51.507270 3817 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:45:51.507348 3817 namespace.go:142] [rhbk-operator] updating namespace I0428 15:45:51.508124 3817 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:51.516796 3817 namespace.go:142] [konflux-ui] updating namespace I0428 15:45:51.521617 3817 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:51.531451 3817 namespace.go:142] [konflux-ui] updating namespace I0428 15:45:51.531471 3817 namespace.go:142] [tsf-tas] updating namespace I0428 15:45:51.531644 3817 namespace.go:142] [konflux-operator] updating namespace I0428 15:45:51.531648 3817 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:45:51.545475 3817 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:51.557784 3817 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:51.562194 3817 namespace.go:142] [rhbk-operator] updating namespace I0428 15:45:51.572429 3817 namespace.go:142] [rhbk-operator] updating namespace I0428 15:45:51.586430 3817 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:51.598507 3817 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:51.598626 3817 namespace.go:142] [tsf-tas] updating namespace I0428 15:45:51.618415 3817 namespace.go:142] [tsf-tas] updating namespace I0428 15:45:51.624418 3817 namespace.go:142] [konflux-operator] updating namespace I0428 15:45:51.630065 3817 namespace.go:142] [konflux-operator] updating namespace I0428 15:45:51.635178 3817 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:45:51.647452 3817 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:45:52.792800 3817 namespace.go:142] [tsf-tpa] updating namespace I0428 15:45:52.806122 3817 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:45:58.291725 3817 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.292275 3817 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.292300 3817 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.292308 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:45:58.292324 3817 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:45:58.292329 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:45:59.322048 3817 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:59.322071 3817 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:59.322085 3817 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:45:59.322094 3817 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa01586jksk of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:00.327510 3817 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.328024 3817 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.328053 3817 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.328063 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:00.328077 3817 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:00.328085 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:01.359581 3817 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:01.359602 3817 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:01.359612 3817 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:01.359623 3817 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19dgh4wr of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:03.893462 3817 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.893924 3817 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.893949 3817 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.893957 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:03.893966 3817 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:03.893970 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:04.927790 3817 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:04.927816 3817 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:04.927831 3817 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:04.927848 3817 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51ngx6w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:05.482384 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0428 15:46:07.657180 3817 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.657714 3817 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.657746 3817 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.657754 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:07.657763 3817 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.657767 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:07.767683 3817 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.768190 3817 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.768213 3817 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.768221 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:07.768230 3817 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:07.768234 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:08.702781 3817 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.702810 3817 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.702825 3817 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.702848 3817 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d76sw5q of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.865762 3817 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.865793 3817 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.865811 3817 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:08.865824 3817 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2kdg2m of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:10.471464 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0428 15:46:10.471775 3817 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.472357 3817 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.472384 3817 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.472395 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0428 15:46:10.472408 3817 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0428 15:46:10.472414 3817 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0428 15:46:11.504790 3817 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:11.504821 3817 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:11.504834 3817 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:11.504850 3817 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136mvngw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:13.713723 3817 namespace.go:98] [cert-manager] adding namespace I0428 15:46:13.714659 3817 namespace.go:102] [cert-manager] adding namespace took 910.738µs I0428 15:46:13.755704 3817 namespace.go:142] [cert-manager] updating namespace I0428 15:46:13.755825 3817 namespace.go:142] [cert-manager] updating namespace I0428 15:46:14.563693 3817 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-h77gq] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-h77gq for pod on switch ip-10-0-0-9.ec2.internal I0428 15:46:14.563789 3817 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-h77gq I0428 15:46:14.585847 3817 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-h77gq] pod update took 22.081999ms I0428 15:46:14.585870 3817 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-h77gq] addLogicalPort annotation time took 22.111457ms I0428 15:46:14.587080 3817 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-h77gq] addLogicalPort took 23.404058ms, libovsdb time 886.194µs I0428 15:46:14.847216 3817 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-rx2z6] creating logical port konflux-operator_konflux-operator-controller-manager-66b879f574-rx2z6 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:46:14.847334 3817 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-66b879f574-rx2z6 I0428 15:46:14.869766 3817 pod.go:62] [konflux-operator/konflux-operator-controller-manager-66b879f574-rx2z6] pod update took 22.459517ms I0428 15:46:14.869789 3817 base_network_controller_pods.go:951] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-rx2z6] addLogicalPort annotation time took 22.486423ms I0428 15:46:14.871095 3817 pods.go:271] [konflux-operator/konflux-operator-controller-manager-66b879f574-rx2z6] addLogicalPort took 23.901586ms, libovsdb time 975.325µs I0428 15:46:14.935740 3817 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-h77gq 7eb7d4286afdfe92435c8d1951dec05ac3665728e0bf64baa631dd77e80dfd4f network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-h77gq 7eb7d4286afdfe92435c8d1951dec05ac3665728e0bf64baa631dd77e80dfd4f network default NAD default NAD key ] I0428 15:46:14.945277 3817 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-h77gq, hostIfaceName: 7eb7d4286afdfe9, network: default, NAD default, SandboxID: "7eb7d4286afdfe92435c8d1951dec05ac3665728e0bf64baa631dd77e80dfd4f", PCI device ID: , UID: "f0c8b3b9-3eef-4fd0-bfa6-2085db730dee", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0428 15:46:14.995718 3817 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-h77gq 7eb7d4286afdfe92435c8d1951dec05ac3665728e0bf64baa631dd77e80dfd4f network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-h77gq 7eb7d4286afdfe92435c8d1951dec05ac3665728e0bf64baa631dd77e80dfd4f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:ef:bf:54:90:74\",\"name\":\"7eb7d4286afdfe9\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fc9d9eb9-942f-43e7-a54b-256e9444e0ad\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:46:15.219600 3817 cni.go:452] [konflux-operator/konflux-operator-controller-manager-66b879f574-rx2z6 b10fb6016cd459525ddba89fceed48f9017ecf848d5346517cfb6d6ff98a38fb network default NAD default NAD key ] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-rx2z6 b10fb6016cd459525ddba89fceed48f9017ecf848d5346517cfb6d6ff98a38fb network default NAD default NAD key ] I0428 15:46:15.231302 3817 helper_linux.go:556] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-66b879f574-rx2z6, hostIfaceName: b10fb6016cd4595, network: default, NAD default, SandboxID: "b10fb6016cd459525ddba89fceed48f9017ecf848d5346517cfb6d6ff98a38fb", PCI device ID: , UID: "b7ec6c26-0dba-404d-8c58-11f20f1b73b0", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0428 15:46:15.277894 3817 cni.go:473] [konflux-operator/konflux-operator-controller-manager-66b879f574-rx2z6 b10fb6016cd459525ddba89fceed48f9017ecf848d5346517cfb6d6ff98a38fb network default NAD default NAD key default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-rx2z6 b10fb6016cd459525ddba89fceed48f9017ecf848d5346517cfb6d6ff98a38fb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:e3:a7:e4:cd:e0\",\"name\":\"b10fb6016cd4595\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ee0b48ae-76d0-4939-a25c-d02bc7ba65f7\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:46:19.240877 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 15:46:19.240900 3817 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 15:46:19.274484 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.893711 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.911641 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.933790 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:21.954617 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0428 15:46:23.371583 3817 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-4b4q4] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-4b4q4 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:46:23.371697 3817 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-4b4q4 I0428 15:46:23.395823 3817 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-4b4q4] pod update took 24.150928ms I0428 15:46:23.395850 3817 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-4b4q4] addLogicalPort annotation time took 24.1828ms I0428 15:46:23.396999 3817 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-4b4q4] addLogicalPort took 25.453973ms, libovsdb time 835.643µs I0428 15:46:23.734204 3817 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-4b4q4 2b20bcbc9c5fe4f61c8f18214b200866365ee6cab4c7549fbd56e6b87ae1649f network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-4b4q4 2b20bcbc9c5fe4f61c8f18214b200866365ee6cab4c7549fbd56e6b87ae1649f network default NAD default NAD key ] I0428 15:46:23.744144 3817 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-4b4q4, hostIfaceName: 2b20bcbc9c5fe4f, network: default, NAD default, SandboxID: "2b20bcbc9c5fe4f61c8f18214b200866365ee6cab4c7549fbd56e6b87ae1649f", PCI device ID: , UID: "f0b7b2a0-f2ab-46b1-b308-b1e670aab6ae", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0428 15:46:23.792749 3817 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-4b4q4 2b20bcbc9c5fe4f61c8f18214b200866365ee6cab4c7549fbd56e6b87ae1649f network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-4b4q4 2b20bcbc9c5fe4f61c8f18214b200866365ee6cab4c7549fbd56e6b87ae1649f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:99:41:54:89:92\",\"name\":\"2b20bcbc9c5fe4f\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ad0c765e-7f26-4c2d-89ef-116f581abeeb\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:46:25.252623 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0428 15:46:25.252644 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0428 15:46:30.106236 3817 namespace.go:142] [openshift-operators] updating namespace I0428 15:46:30.114776 3817 namespace.go:142] [openshift-operators] updating namespace W0428 15:46:30.244111 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0428 15:46:36.085361 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0428 15:46:37.475405 3817 namespace.go:142] [openshift-operators] updating namespace I0428 15:46:37.746579 3817 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-r5zr4] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-r5zr4 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:46:37.746697 3817 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-r5zr4 I0428 15:46:37.766187 3817 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-r5zr4] pod update took 19.511654ms I0428 15:46:37.766217 3817 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-r5zr4] addLogicalPort annotation time took 19.545946ms I0428 15:46:37.767596 3817 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-r5zr4] addLogicalPort took 21.053365ms, libovsdb time 968.878µs I0428 15:46:38.115021 3817 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-r5zr4 43023d1136276cc1c8c8434c8572afafcfab297f01476401be127a9312028dce network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-r5zr4 43023d1136276cc1c8c8434c8572afafcfab297f01476401be127a9312028dce network default NAD default NAD key ] I0428 15:46:38.124575 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-r5zr4, hostIfaceName: 43023d1136276cc, network: default, NAD default, SandboxID: "43023d1136276cc1c8c8434c8572afafcfab297f01476401be127a9312028dce", PCI device ID: , UID: "94c6de01-7351-42dd-9dae-ac6fd2b3f4a0", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0428 15:46:38.178487 3817 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-r5zr4 43023d1136276cc1c8c8434c8572afafcfab297f01476401be127a9312028dce network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-r5zr4 43023d1136276cc1c8c8434c8572afafcfab297f01476401be127a9312028dce network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:d9:12:24:9f:c5\",\"name\":\"43023d1136276cc\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d0d53ce7-c77d-4d3e-92c9-b99c13642dd7\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:46:39.375372 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=78 I0428 15:46:42.052361 3817 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-kbd4g] creating logical port openshift-operators_rhtas-operator-controller-manager-68888cb75-kbd4g for pod on switch ip-10-0-0-9.ec2.internal I0428 15:46:42.052476 3817 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-68888cb75-kbd4g I0428 15:46:42.095036 3817 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-68888cb75-kbd4g] pod update took 42.584901ms I0428 15:46:42.095062 3817 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-kbd4g] addLogicalPort annotation time took 42.615974ms I0428 15:46:42.096298 3817 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-68888cb75-kbd4g] addLogicalPort took 43.952813ms, libovsdb time 866.502µs I0428 15:46:42.410510 3817 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-68888cb75-kbd4g 1bce79f926a8798b4b35dfcb0badb81f2861cfe349d0290f00502162d951f32e network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-kbd4g 1bce79f926a8798b4b35dfcb0badb81f2861cfe349d0290f00502162d951f32e network default NAD default NAD key ] I0428 15:46:42.421347 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-68888cb75-kbd4g, hostIfaceName: 1bce79f926a8798, network: default, NAD default, SandboxID: "1bce79f926a8798b4b35dfcb0badb81f2861cfe349d0290f00502162d951f32e", PCI device ID: , UID: "0e2b9fc0-9d6a-4648-818c-c239ca15d651", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0428 15:46:42.471825 3817 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-68888cb75-kbd4g 1bce79f926a8798b4b35dfcb0badb81f2861cfe349d0290f00502162d951f32e network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-kbd4g 1bce79f926a8798b4b35dfcb0badb81f2861cfe349d0290f00502162d951f32e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:54:f2:20:e1:58\",\"name\":\"1bce79f926a8798\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/adf53dc2-47ea-4c74-a261-c9295d34f304\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:46:43.384806 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=623 I0428 15:46:44.979485 3817 namespace.go:98] [trusted-artifact-signer] adding namespace I0428 15:46:44.980328 3817 namespace.go:102] [trusted-artifact-signer] adding namespace took 820.183µs I0428 15:46:45.004101 3817 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:45.024415 3817 namespace.go:142] [trusted-artifact-signer] updating namespace W0428 15:46:45.383411 3817 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0428 15:46:50.659174 3817 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:50.659727 3817 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:46:50.680119 3817 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:50.680140 3817 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:46:50.686086 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0428 15:46:50.686109 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0428 15:46:53.211828 3817 namespace.go:98] [openshift-pipelines] adding namespace I0428 15:46:53.212786 3817 namespace.go:102] [openshift-pipelines] adding namespace took 933.632µs I0428 15:46:53.234233 3817 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:46:53.940012 3817 namespace.go:142] [cert-manager] updating namespace I0428 15:46:54.142192 3817 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:46:54.547123 3817 namespace.go:142] [konflux-operator] updating namespace I0428 15:46:54.746616 3817 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:54.947335 3817 namespace.go:142] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] updating namespace I0428 15:46:55.349221 3817 namespace.go:142] [rhbk-operator] updating namespace I0428 15:46:55.551552 3817 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:46:55.750665 3817 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:55.951684 3817 namespace.go:142] [tsf] updating namespace I0428 15:46:56.154569 3817 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:46:56.354115 3817 namespace.go:142] [tsf-tas] updating namespace W0428 15:46:56.371371 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0428 15:46:56.556873 3817 namespace.go:142] [tsf-tpa] updating namespace I0428 15:46:56.689766 3817 namespace.go:142] [cert-manager] updating namespace I0428 15:46:56.702204 3817 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:46:56.714869 3817 namespace.go:142] [default] updating namespace I0428 15:46:56.728078 3817 namespace.go:142] [konflux-operator] updating namespace I0428 15:46:56.737681 3817 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:56.751515 3817 namespace.go:142] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] updating namespace I0428 15:46:56.767940 3817 namespace.go:142] [openshift] updating namespace I0428 15:46:56.785037 3817 namespace.go:142] [rhbk-operator] updating namespace I0428 15:46:56.808961 3817 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:46:56.827523 3817 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:56.847595 3817 namespace.go:142] [tsf] updating namespace I0428 15:46:56.864868 3817 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:46:56.890514 3817 namespace.go:142] [tsf-tas] updating namespace I0428 15:46:56.919449 3817 namespace.go:142] [tsf-tpa] updating namespace I0428 15:46:57.015632 3817 namespace.go:142] [cert-manager] updating namespace I0428 15:46:57.106705 3817 namespace.go:142] [cert-manager-operator] updating namespace I0428 15:46:57.175246 3817 namespace.go:142] [default] updating namespace I0428 15:46:57.241183 3817 namespace.go:142] [konflux-operator] updating namespace I0428 15:46:57.347542 3817 namespace.go:142] [konflux-ui] updating namespace I0428 15:46:57.447649 3817 namespace.go:142] [open-cluster-management-2pv84nia58gje8norbmmjqtj26oprnd8] updating namespace I0428 15:46:57.547812 3817 namespace.go:142] [openshift] updating namespace I0428 15:46:57.648575 3817 namespace.go:142] [rhbk-operator] updating namespace I0428 15:46:57.745546 3817 namespace.go:142] [rhtpa-operator] updating namespace I0428 15:46:57.843635 3817 namespace.go:142] [trusted-artifact-signer] updating namespace I0428 15:46:57.945552 3817 namespace.go:142] [tsf] updating namespace I0428 15:46:58.045737 3817 namespace.go:142] [tsf-keycloak] updating namespace I0428 15:46:58.146343 3817 namespace.go:142] [tsf-tas] updating namespace I0428 15:46:58.247338 3817 namespace.go:142] [tsf-tpa] updating namespace I0428 15:46:58.662236 3817 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:47:06.186686 3817 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-9xxsp] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-9xxsp for pod on switch ip-10-0-0-9.ec2.internal I0428 15:47:06.186833 3817 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-9xxsp I0428 15:47:06.201883 3817 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-9xxsp] pod update took 15.087393ms I0428 15:47:06.201906 3817 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-9xxsp] addLogicalPort annotation time took 15.123468ms I0428 15:47:06.203405 3817 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-9xxsp] addLogicalPort took 16.733451ms, libovsdb time 993.697µs I0428 15:47:06.374523 3817 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h] creating logical port openshift-pipelines_tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h for pod on switch ip-10-0-0-9.ec2.internal I0428 15:47:06.374660 3817 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h I0428 15:47:06.389887 3817 pod.go:62] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h] pod update took 15.26579ms I0428 15:47:06.389923 3817 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h] addLogicalPort annotation time took 15.313531ms I0428 15:47:06.391276 3817 pods.go:271] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h] addLogicalPort took 16.769402ms, libovsdb time 885.067µs I0428 15:47:06.484528 3817 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-kl6g7] creating logical port openshift-pipelines_tekton-pipelines-webhook-7b6d6f4c94-kl6g7 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:47:06.484654 3817 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-kl6g7 I0428 15:47:06.500816 3817 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-kl6g7] pod update took 16.195ms I0428 15:47:06.500845 3817 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-kl6g7] addLogicalPort annotation time took 16.235674ms I0428 15:47:06.502524 3817 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-kl6g7] addLogicalPort took 18.014851ms, libovsdb time 929.299µs I0428 15:47:06.557048 3817 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-9xxsp d67d78f478b36c94cc61acd9a6d503c88d513caabf401e59675a490a1f40b5d9 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-9xxsp d67d78f478b36c94cc61acd9a6d503c88d513caabf401e59675a490a1f40b5d9 network default NAD default NAD key ] I0428 15:47:06.565308 3817 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-rzn9n] creating logical port openshift-pipelines_tekton-operator-proxy-webhook-57f84cff6f-rzn9n for pod on switch ip-10-0-0-9.ec2.internal I0428 15:47:06.565485 3817 kube.go:256] Updating pod openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-rzn9n I0428 15:47:06.567315 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-9xxsp, hostIfaceName: d67d78f478b36c9, network: default, NAD default, SandboxID: "d67d78f478b36c94cc61acd9a6d503c88d513caabf401e59675a490a1f40b5d9", PCI device ID: , UID: "9ab30cfd-40aa-4e7f-81f7-6556f85ff9e1", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0428 15:47:06.581054 3817 pod.go:62] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-rzn9n] pod update took 15.607551ms I0428 15:47:06.581078 3817 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-rzn9n] addLogicalPort annotation time took 15.637002ms I0428 15:47:06.582503 3817 pods.go:271] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-rzn9n] addLogicalPort took 17.213174ms, libovsdb time 900.931µs W0428 15:47:06.604454 3817 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql I0428 15:47:06.621085 3817 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-9xxsp d67d78f478b36c94cc61acd9a6d503c88d513caabf401e59675a490a1f40b5d9 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-9xxsp d67d78f478b36c94cc61acd9a6d503c88d513caabf401e59675a490a1f40b5d9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2a:06:a8:ff:36:76\",\"name\":\"d67d78f478b36c9\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3d6611ab-0cc4-423c-8210-7b4f4b0a075c\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:47:06.630876 3817 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0428 15:47:06.736755 3817 cni.go:452] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h 3f0658eabece86fee0f56762e1cc4d4bf002a2e76ff9a1d61c238c19fc6217bf network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h 3f0658eabece86fee0f56762e1cc4d4bf002a2e76ff9a1d61c238c19fc6217bf network default NAD default NAD key ] I0428 15:47:06.751242 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h, hostIfaceName: 3f0658eabece86f, network: default, NAD default, SandboxID: "3f0658eabece86fee0f56762e1cc4d4bf002a2e76ff9a1d61c238c19fc6217bf", PCI device ID: , UID: "1d4356d2-35f1-4644-8672-813bd773b9bd", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0428 15:47:06.805761 3817 cni.go:473] [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h 3f0658eabece86fee0f56762e1cc4d4bf002a2e76ff9a1d61c238c19fc6217bf network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-remote-resolvers-85f5bb4dfb-r8h9h 3f0658eabece86fee0f56762e1cc4d4bf002a2e76ff9a1d61c238c19fc6217bf network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:23:2e:96:a5:26\",\"name\":\"3f0658eabece86f\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/34cf6ca2-670a-40be-9df5-e1b5bf3d86de\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:47:06.850078 3817 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-kl6g7 185fb9357b58167e3872f4b8f3f40e0e3e7c4a1616c1b20b7e185705426d397c network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-kl6g7 185fb9357b58167e3872f4b8f3f40e0e3e7c4a1616c1b20b7e185705426d397c network default NAD default NAD key ] I0428 15:47:06.859623 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-7b6d6f4c94-kl6g7, hostIfaceName: 185fb9357b58167, network: default, NAD default, SandboxID: "185fb9357b58167e3872f4b8f3f40e0e3e7c4a1616c1b20b7e185705426d397c", PCI device ID: , UID: "473717c6-0410-4b13-866a-ba52e3ba23c3", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0428 15:47:06.913107 3817 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-kl6g7 185fb9357b58167e3872f4b8f3f40e0e3e7c4a1616c1b20b7e185705426d397c network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-kl6g7 185fb9357b58167e3872f4b8f3f40e0e3e7c4a1616c1b20b7e185705426d397c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:51:95:e1:64:d2\",\"name\":\"185fb9357b58167\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1bd66f86-6a6f-4916-b01b-3c090994ad02\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:47:06.933756 3817 cni.go:452] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-rzn9n dd45c1db3cecadfea6dd2019370bd631c3a5cca60f2c0af5eed71c8db1c1c6a0 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-rzn9n dd45c1db3cecadfea6dd2019370bd631c3a5cca60f2c0af5eed71c8db1c1c6a0 network default NAD default NAD key ] I0428 15:47:06.943977 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-operator-proxy-webhook-57f84cff6f-rzn9n, hostIfaceName: dd45c1db3cecadf, network: default, NAD default, SandboxID: "dd45c1db3cecadfea6dd2019370bd631c3a5cca60f2c0af5eed71c8db1c1c6a0", PCI device ID: , UID: "2c051858-3aea-4989-9670-dae408b6fdd0", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0428 15:47:06.994786 3817 cni.go:473] [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-rzn9n dd45c1db3cecadfea6dd2019370bd631c3a5cca60f2c0af5eed71c8db1c1c6a0 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-operator-proxy-webhook-57f84cff6f-rzn9n dd45c1db3cecadfea6dd2019370bd631c3a5cca60f2c0af5eed71c8db1c1c6a0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:38:1d:4e:e2:e3\",\"name\":\"dd45c1db3cecadf\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/99738940-2ba8-46a8-bf4f-06802f736540\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:47:10.210615 3817 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:10.210638 3817 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:10.210645 3817 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0428 15:47:10.487794 3817 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:10.487815 3817 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:10.487822 3817 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:10.487827 3817 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:13.505871 3817 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:13.505896 3817 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:13.505905 3817 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0428 15:47:16.372782 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=70 W0428 15:47:21.498812 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:21.498841 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:21.498850 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0428 15:47:21.498856 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook I0428 15:47:24.465997 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 W0428 15:47:24.516722 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:24.516743 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0428 15:47:24.516748 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0428 15:47:31.209318 3817 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-controller-785c845664-h7rhq] creating logical port openshift-pipelines_tekton-triggers-controller-785c845664-h7rhq for pod on switch ip-10-0-0-9.ec2.internal I0428 15:47:31.209412 3817 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-controller-785c845664-h7rhq I0428 15:47:31.227348 3817 pod.go:62] [openshift-pipelines/tekton-triggers-controller-785c845664-h7rhq] pod update took 17.966458ms I0428 15:47:31.227373 3817 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-controller-785c845664-h7rhq] addLogicalPort annotation time took 17.995145ms I0428 15:47:31.228699 3817 pods.go:271] [openshift-pipelines/tekton-triggers-controller-785c845664-h7rhq] addLogicalPort took 19.396947ms, libovsdb time 909.868µs I0428 15:47:31.567757 3817 cni.go:452] [openshift-pipelines/tekton-triggers-controller-785c845664-h7rhq 8a9c5a8cab769ecc4b878d778b519b1d944e7f86cc95accee2a8f0c72c913666 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-h7rhq 8a9c5a8cab769ecc4b878d778b519b1d944e7f86cc95accee2a8f0c72c913666 network default NAD default NAD key ] I0428 15:47:31.576866 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-controller-785c845664-h7rhq, hostIfaceName: 8a9c5a8cab769ec, network: default, NAD default, SandboxID: "8a9c5a8cab769ecc4b878d778b519b1d944e7f86cc95accee2a8f0c72c913666", PCI device ID: , UID: "bc32dd85-3244-407c-9f22-53ba5fbd823a", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0428 15:47:31.629050 3817 cni.go:473] [openshift-pipelines/tekton-triggers-controller-785c845664-h7rhq 8a9c5a8cab769ecc4b878d778b519b1d944e7f86cc95accee2a8f0c72c913666 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-controller-785c845664-h7rhq 8a9c5a8cab769ecc4b878d778b519b1d944e7f86cc95accee2a8f0c72c913666 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ca:91:5a:a8:0c:94\",\"name\":\"8a9c5a8cab769ec\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/370b1a4b-3c34-45c4-8cbe-e63792c00477\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:47:32.492605 3817 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0428 15:47:35.486664 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=11 W0428 15:47:36.326959 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0428 15:47:36.891769 3817 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:36.892327 3817 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:47:36.912389 3817 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:36.912414 3817 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:47:36.915203 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0428 15:47:36.915233 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0428 15:47:38.654548 3817 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0428 15:47:38.655211 3817 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0428 15:47:38.655231 3817 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0428 15:47:38.657478 3817 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0428 15:47:44.498072 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 W0428 15:47:46.815646 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0428 15:47:46.815665 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0428 15:47:46.816489 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0428 15:47:46.816505 3817 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0428 15:47:54.268722 3817 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0428 15:48:07.859210 3817 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-64c5c8474c-pjgt7] creating logical port openshift-pipelines_tekton-results-api-64c5c8474c-pjgt7 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:48:07.859337 3817 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-64c5c8474c-pjgt7 I0428 15:48:07.875613 3817 pod.go:62] [openshift-pipelines/tekton-results-api-64c5c8474c-pjgt7] pod update took 16.302446ms I0428 15:48:07.875638 3817 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-api-64c5c8474c-pjgt7] addLogicalPort annotation time took 16.33209ms I0428 15:48:07.876997 3817 pods.go:271] [openshift-pipelines/tekton-results-api-64c5c8474c-pjgt7] addLogicalPort took 17.804526ms, libovsdb time 956.241µs I0428 15:48:08.224836 3817 cni.go:452] [openshift-pipelines/tekton-results-api-64c5c8474c-pjgt7 b0a829dc3fe6ec165be62e421717b35b5fb9e439787f069c6c8948f8ed3255f0 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-pjgt7 b0a829dc3fe6ec165be62e421717b35b5fb9e439787f069c6c8948f8ed3255f0 network default NAD default NAD key ] I0428 15:48:08.234032 3817 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-64c5c8474c-pjgt7, hostIfaceName: b0a829dc3fe6ec1, network: default, NAD default, SandboxID: "b0a829dc3fe6ec165be62e421717b35b5fb9e439787f069c6c8948f8ed3255f0", PCI device ID: , UID: "59a19528-50e0-47f2-9fe1-715dc5680384", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0428 15:48:08.284450 3817 cni.go:473] [openshift-pipelines/tekton-results-api-64c5c8474c-pjgt7 b0a829dc3fe6ec165be62e421717b35b5fb9e439787f069c6c8948f8ed3255f0 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-pjgt7 b0a829dc3fe6ec165be62e421717b35b5fb9e439787f069c6c8948f8ed3255f0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:f0:c4:f7:33:42\",\"name\":\"b0a829dc3fe6ec1\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b10e2027-6078-4d64-8dce-146770448587\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:48:13.797413 3817 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:48:13.797435 3817 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:48:13.797444 3817 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:48:18.827610 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:48:18.827630 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0428 15:48:18.827636 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0428 15:48:20.455605 3817 namespace.go:142] [openshift-pipelines] updating namespace I0428 15:48:29.233215 3817 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:29.233792 3817 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:29.269215 3817 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:29.269234 3817 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:29.274761 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0428 15:48:29.274785 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0428 15:48:30.951782 3817 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-p7x8d-22qzd] creating logical port tsf-tas_segment-backup-installation-p7x8d-22qzd for pod on switch ip-10-0-0-9.ec2.internal I0428 15:48:30.951870 3817 kube.go:256] Updating pod tsf-tas/segment-backup-installation-p7x8d-22qzd I0428 15:48:30.964415 3817 pod.go:62] [tsf-tas/segment-backup-installation-p7x8d-22qzd] pod update took 12.565927ms I0428 15:48:30.964448 3817 base_network_controller_pods.go:951] [default/tsf-tas/segment-backup-installation-p7x8d-22qzd] addLogicalPort annotation time took 12.603578ms I0428 15:48:30.965994 3817 pods.go:271] [tsf-tas/segment-backup-installation-p7x8d-22qzd] addLogicalPort took 14.231006ms, libovsdb time 1.022964ms I0428 15:48:31.314191 3817 cni.go:452] [tsf-tas/segment-backup-installation-p7x8d-22qzd df7aa67e2edcfd1c271cf6380b84be57b43ffba3c55718e02e98d73df545774c network default NAD default NAD key ] ADD starting CNI request [tsf-tas/segment-backup-installation-p7x8d-22qzd df7aa67e2edcfd1c271cf6380b84be57b43ffba3c55718e02e98d73df545774c network default NAD default NAD key ] I0428 15:48:31.324236 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-p7x8d-22qzd, hostIfaceName: df7aa67e2edcfd1, network: default, NAD default, SandboxID: "df7aa67e2edcfd1c271cf6380b84be57b43ffba3c55718e02e98d73df545774c", PCI device ID: , UID: "f6c14faf-0b09-4bf0-8277-7925a7312959", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0428 15:48:31.379728 3817 cni.go:473] [tsf-tas/segment-backup-installation-p7x8d-22qzd df7aa67e2edcfd1c271cf6380b84be57b43ffba3c55718e02e98d73df545774c network default NAD default NAD key default] ADD finished CNI request [tsf-tas/segment-backup-installation-p7x8d-22qzd df7aa67e2edcfd1c271cf6380b84be57b43ffba3c55718e02e98d73df545774c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:f2:1c:07:ae:70\",\"name\":\"df7aa67e2edcfd1\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a7526a56-65a5-478e-9fc7-fc920bd8ba0a\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:48:31.730485 3817 base_network_controller_pods.go:487] [default/tsf-tas/ctlog-createtree-job-9pz4t-55l6n] creating logical port tsf-tas_ctlog-createtree-job-9pz4t-55l6n for pod on switch ip-10-0-0-9.ec2.internal I0428 15:48:31.730599 3817 kube.go:256] Updating pod tsf-tas/ctlog-createtree-job-9pz4t-55l6n I0428 15:48:31.746877 3817 pod.go:62] [tsf-tas/ctlog-createtree-job-9pz4t-55l6n] pod update took 16.32155ms I0428 15:48:31.746907 3817 base_network_controller_pods.go:951] [default/tsf-tas/ctlog-createtree-job-9pz4t-55l6n] addLogicalPort annotation time took 16.355162ms I0428 15:48:31.748355 3817 pods.go:271] [tsf-tas/ctlog-createtree-job-9pz4t-55l6n] addLogicalPort took 17.887395ms, libovsdb time 911.632µs I0428 15:48:31.836818 3817 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logserver-7c6fc8f7bb-c6b2h] creating logical port tsf-tas_trillian-logserver-7c6fc8f7bb-c6b2h for pod on switch ip-10-0-0-9.ec2.internal I0428 15:48:31.836948 3817 kube.go:256] Updating pod tsf-tas/trillian-logserver-7c6fc8f7bb-c6b2h I0428 15:48:31.856183 3817 pod.go:62] [tsf-tas/trillian-logserver-7c6fc8f7bb-c6b2h] pod update took 19.272074ms I0428 15:48:31.856215 3817 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logserver-7c6fc8f7bb-c6b2h] addLogicalPort annotation time took 19.313467ms I0428 15:48:31.857825 3817 pods.go:271] [tsf-tas/trillian-logserver-7c6fc8f7bb-c6b2h] addLogicalPort took 21.028998ms, libovsdb time 1.031516ms I0428 15:48:32.094331 3817 cni.go:452] [tsf-tas/ctlog-createtree-job-9pz4t-55l6n 2c43ec73f25b882a87d24749d0ace39b93649065db7ec2e55cc7e49061d16a9d network default NAD default NAD key ] ADD starting CNI request [tsf-tas/ctlog-createtree-job-9pz4t-55l6n 2c43ec73f25b882a87d24749d0ace39b93649065db7ec2e55cc7e49061d16a9d network default NAD default NAD key ] I0428 15:48:32.104777 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: ctlog-createtree-job-9pz4t-55l6n, hostIfaceName: 2c43ec73f25b882, network: default, NAD default, SandboxID: "2c43ec73f25b882a87d24749d0ace39b93649065db7ec2e55cc7e49061d16a9d", PCI device ID: , UID: "c38e99dc-4aeb-4bc0-b30a-b5b83c717118", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0428 15:48:32.161880 3817 cni.go:473] [tsf-tas/ctlog-createtree-job-9pz4t-55l6n 2c43ec73f25b882a87d24749d0ace39b93649065db7ec2e55cc7e49061d16a9d network default NAD default NAD key default] ADD finished CNI request [tsf-tas/ctlog-createtree-job-9pz4t-55l6n 2c43ec73f25b882a87d24749d0ace39b93649065db7ec2e55cc7e49061d16a9d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:23:24:58:dd:3d\",\"name\":\"2c43ec73f25b882\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/93bf3082-f280-4ef2-b696-a57d9a80b74a\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:48:32.807668 3817 cni.go:452] [tsf-tas/trillian-logserver-7c6fc8f7bb-c6b2h 39e9de31ec9eab6bbd2f3470a7a16ad5254e7d9d80021e3950f07eafdf8dac7a network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logserver-7c6fc8f7bb-c6b2h 39e9de31ec9eab6bbd2f3470a7a16ad5254e7d9d80021e3950f07eafdf8dac7a network default NAD default NAD key ] I0428 15:48:32.821797 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logserver-7c6fc8f7bb-c6b2h, hostIfaceName: 39e9de31ec9eab6, network: default, NAD default, SandboxID: "39e9de31ec9eab6bbd2f3470a7a16ad5254e7d9d80021e3950f07eafdf8dac7a", PCI device ID: , UID: "d7f0bec7-5b30-4902-a01f-6a857622195b", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0428 15:48:32.900469 3817 cni.go:473] [tsf-tas/trillian-logserver-7c6fc8f7bb-c6b2h 39e9de31ec9eab6bbd2f3470a7a16ad5254e7d9d80021e3950f07eafdf8dac7a network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logserver-7c6fc8f7bb-c6b2h 39e9de31ec9eab6bbd2f3470a7a16ad5254e7d9d80021e3950f07eafdf8dac7a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:c5:fb:48:eb:b0\",\"name\":\"39e9de31ec9eab6\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7c9e4a50-3f88-4b14-be9a-39a7f2a7c172\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:48:34.625262 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0428 15:48:34.625362 3817 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server I0428 15:48:34.711922 3817 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-646864b544-5tqr8] creating logical port tsf-tas_trillian-db-646864b544-5tqr8 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:48:34.712029 3817 kube.go:256] Updating pod tsf-tas/trillian-db-646864b544-5tqr8 I0428 15:48:34.725668 3817 pod.go:62] [tsf-tas/trillian-db-646864b544-5tqr8] pod update took 13.659328ms I0428 15:48:34.725764 3817 base_network_controller_pods.go:951] [default/tsf-tas/trillian-db-646864b544-5tqr8] addLogicalPort annotation time took 13.760084ms I0428 15:48:34.727498 3817 pods.go:271] [tsf-tas/trillian-db-646864b544-5tqr8] addLogicalPort took 15.597157ms, libovsdb time 1.18113ms I0428 15:48:38.293579 3817 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-m7t22 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:38.294258 3817 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-m7t22 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:39.175206 3817 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-m7t22 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:39.175237 3817 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-m7t22 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:48:42.178507 3817 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0428 15:48:44.195338 3817 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0428 15:48:44.631458 3817 cni.go:452] [tsf-tas/trillian-db-646864b544-5tqr8 aa91c55d24d8d5c91d8b6f724286964daefa0f6893b9496e3d8e23e5782afc6f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-db-646864b544-5tqr8 aa91c55d24d8d5c91d8b6f724286964daefa0f6893b9496e3d8e23e5782afc6f network default NAD default NAD key ] I0428 15:48:44.640696 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-646864b544-5tqr8, hostIfaceName: aa91c55d24d8d5c, network: default, NAD default, SandboxID: "aa91c55d24d8d5c91d8b6f724286964daefa0f6893b9496e3d8e23e5782afc6f", PCI device ID: , UID: "02c4abe3-ee54-404b-96bb-58a2da0ddb40", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0428 15:48:44.697606 3817 cni.go:473] [tsf-tas/trillian-db-646864b544-5tqr8 aa91c55d24d8d5c91d8b6f724286964daefa0f6893b9496e3d8e23e5782afc6f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-db-646864b544-5tqr8 aa91c55d24d8d5c91d8b6f724286964daefa0f6893b9496e3d8e23e5782afc6f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:64:a6:94:d7:dd\",\"name\":\"aa91c55d24d8d5c\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c62f645e-921f-42b0-9adb-7c9e6969385d\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:48:44.974736 3817 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver W0428 15:48:47.989136 3817 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql I0428 15:48:49.005296 3817 cni.go:452] [tsf-tas/segment-backup-installation-p7x8d-22qzd df7aa67e2edcfd1c271cf6380b84be57b43ffba3c55718e02e98d73df545774c network default NAD default NAD key ] DEL starting CNI request [tsf-tas/segment-backup-installation-p7x8d-22qzd df7aa67e2edcfd1c271cf6380b84be57b43ffba3c55718e02e98d73df545774c network default NAD default NAD key ] I0428 15:48:49.099918 3817 cni.go:473] [tsf-tas/segment-backup-installation-p7x8d-22qzd df7aa67e2edcfd1c271cf6380b84be57b43ffba3c55718e02e98d73df545774c network default NAD default NAD key default] DEL finished CNI request [tsf-tas/segment-backup-installation-p7x8d-22qzd df7aa67e2edcfd1c271cf6380b84be57b43ffba3c55718e02e98d73df545774c network default NAD default NAD key default], result "{}", err I0428 15:48:49.149005 3817 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:49.149046 3817 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-p7x8d-22qzd I0428 15:48:49.150606 3817 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-p7x8d-22qzd, ips: 10.129.0.33 I0428 15:48:49.150646 3817 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:48:50.031941 3817 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:48:50.031973 3817 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-p7x8d-22qzd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:48:59.047371 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql W0428 15:49:14.081896 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logserver I0428 15:49:19.528982 3817 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:19.529603 3817 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:19.548578 3817 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:19.548602 3817 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:19.553719 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0428 15:49:19.553745 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0428 15:49:20.212165 3817 base_network_controller_pods.go:487] [default/tsf/patch-tekton-config-2rx2d] creating logical port tsf_patch-tekton-config-2rx2d for pod on switch ip-10-0-0-9.ec2.internal I0428 15:49:20.212296 3817 kube.go:256] Updating pod tsf/patch-tekton-config-2rx2d I0428 15:49:20.225840 3817 pod.go:62] [tsf/patch-tekton-config-2rx2d] pod update took 13.572481ms I0428 15:49:20.225869 3817 base_network_controller_pods.go:951] [default/tsf/patch-tekton-config-2rx2d] addLogicalPort annotation time took 13.639118ms I0428 15:49:20.227288 3817 pods.go:271] [tsf/patch-tekton-config-2rx2d] addLogicalPort took 15.142262ms, libovsdb time 1.021711ms I0428 15:49:20.636905 3817 cni.go:452] [tsf/patch-tekton-config-2rx2d e1172415c904dce2d51eb51469e32d9019edc202fc7b60c8ccf601b353fbe6d7 network default NAD default NAD key ] ADD starting CNI request [tsf/patch-tekton-config-2rx2d e1172415c904dce2d51eb51469e32d9019edc202fc7b60c8ccf601b353fbe6d7 network default NAD default NAD key ] I0428 15:49:20.646300 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: patch-tekton-config-2rx2d, hostIfaceName: e1172415c904dce, network: default, NAD default, SandboxID: "e1172415c904dce2d51eb51469e32d9019edc202fc7b60c8ccf601b353fbe6d7", PCI device ID: , UID: "f62a4507-5c7d-4e35-8d66-830046f158f6", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0428 15:49:20.700676 3817 cni.go:473] [tsf/patch-tekton-config-2rx2d e1172415c904dce2d51eb51469e32d9019edc202fc7b60c8ccf601b353fbe6d7 network default NAD default NAD key default] ADD finished CNI request [tsf/patch-tekton-config-2rx2d e1172415c904dce2d51eb51469e32d9019edc202fc7b60c8ccf601b353fbe6d7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:32:01:11:ea:d1\",\"name\":\"e1172415c904dce\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fe116df8-c637-43a3-aeca-4b5ce1a66a42\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:49:28.019307 3817 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.019940 3817 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.205642 3817 cni.go:452] [tsf/patch-tekton-config-2rx2d e1172415c904dce2d51eb51469e32d9019edc202fc7b60c8ccf601b353fbe6d7 network default NAD default NAD key ] DEL starting CNI request [tsf/patch-tekton-config-2rx2d e1172415c904dce2d51eb51469e32d9019edc202fc7b60c8ccf601b353fbe6d7 network default NAD default NAD key ] I0428 15:49:28.300626 3817 cni.go:473] [tsf/patch-tekton-config-2rx2d e1172415c904dce2d51eb51469e32d9019edc202fc7b60c8ccf601b353fbe6d7 network default NAD default NAD key default] DEL finished CNI request [tsf/patch-tekton-config-2rx2d e1172415c904dce2d51eb51469e32d9019edc202fc7b60c8ccf601b353fbe6d7 network default NAD default NAD key default], result "{}", err I0428 15:49:28.348333 3817 obj_retry.go:498] Detected object tsf/patch-tekton-config-2rx2d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.348367 3817 pods.go:176] Deleting pod: tsf/patch-tekton-config-2rx2d I0428 15:49:28.349707 3817 pods.go:236] Attempting to release IPs for pod: tsf/patch-tekton-config-2rx2d, ips: 10.129.0.37 I0428 15:49:28.349735 3817 obj_retry.go:498] Detected object tsf/patch-tekton-config-2rx2d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:28.905860 3817 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:28.905885 3817 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-2ngdk-wk5k9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:29.377366 3817 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:29.377396 3817 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:29.505245 3817 base_network_controller_pods.go:487] [default/tsf/tsf-tekton-configuration-5wjr8] creating logical port tsf_tsf-tekton-configuration-5wjr8 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:49:29.505337 3817 kube.go:256] Updating pod tsf/tsf-tekton-configuration-5wjr8 I0428 15:49:29.516483 3817 pod.go:62] [tsf/tsf-tekton-configuration-5wjr8] pod update took 11.167898ms I0428 15:49:29.516503 3817 base_network_controller_pods.go:951] [default/tsf/tsf-tekton-configuration-5wjr8] addLogicalPort annotation time took 11.193085ms I0428 15:49:29.517986 3817 pods.go:271] [tsf/tsf-tekton-configuration-5wjr8] addLogicalPort took 12.756992ms, libovsdb time 1.065781ms I0428 15:49:29.868192 3817 cni.go:452] [tsf/tsf-tekton-configuration-5wjr8 4a2fb3b97735963f898b1eb32f7107361473a24dd95d0bf366c58101b4b3fa28 network default NAD default NAD key ] ADD starting CNI request [tsf/tsf-tekton-configuration-5wjr8 4a2fb3b97735963f898b1eb32f7107361473a24dd95d0bf366c58101b4b3fa28 network default NAD default NAD key ] I0428 15:49:29.877914 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf, podName: tsf-tekton-configuration-5wjr8, hostIfaceName: 4a2fb3b97735963, network: default, NAD default, SandboxID: "4a2fb3b97735963f898b1eb32f7107361473a24dd95d0bf366c58101b4b3fa28", PCI device ID: , UID: "c9c942ce-56e4-407e-ab64-cdc79dae1fa1", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0428 15:49:29.930337 3817 cni.go:473] [tsf/tsf-tekton-configuration-5wjr8 4a2fb3b97735963f898b1eb32f7107361473a24dd95d0bf366c58101b4b3fa28 network default NAD default NAD key default] ADD finished CNI request [tsf/tsf-tekton-configuration-5wjr8 4a2fb3b97735963f898b1eb32f7107361473a24dd95d0bf366c58101b4b3fa28 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:bb:0d:53:6d:b3\",\"name\":\"4a2fb3b97735963\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/92528f92-377d-4b28-b6f1-1250697d0f64\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:49:32.863830 3817 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-644cd7d779-z6tng] creating logical port tsf-tas_rekor-redis-644cd7d779-z6tng for pod on switch ip-10-0-0-9.ec2.internal I0428 15:49:32.863951 3817 kube.go:256] Updating pod tsf-tas/rekor-redis-644cd7d779-z6tng I0428 15:49:32.883054 3817 pod.go:62] [tsf-tas/rekor-redis-644cd7d779-z6tng] pod update took 19.13824ms I0428 15:49:32.883081 3817 base_network_controller_pods.go:951] [default/tsf-tas/rekor-redis-644cd7d779-z6tng] addLogicalPort annotation time took 19.174381ms I0428 15:49:32.884363 3817 pods.go:271] [tsf-tas/rekor-redis-644cd7d779-z6tng] addLogicalPort took 20.557007ms, libovsdb time 853.403µs I0428 15:49:33.236442 3817 cni.go:452] [tsf-tas/ctlog-createtree-job-9pz4t-55l6n 2c43ec73f25b882a87d24749d0ace39b93649065db7ec2e55cc7e49061d16a9d network default NAD default NAD key ] DEL starting CNI request [tsf-tas/ctlog-createtree-job-9pz4t-55l6n 2c43ec73f25b882a87d24749d0ace39b93649065db7ec2e55cc7e49061d16a9d network default NAD default NAD key ] I0428 15:49:33.322789 3817 cni.go:473] [tsf-tas/ctlog-createtree-job-9pz4t-55l6n 2c43ec73f25b882a87d24749d0ace39b93649065db7ec2e55cc7e49061d16a9d network default NAD default NAD key default] DEL finished CNI request [tsf-tas/ctlog-createtree-job-9pz4t-55l6n 2c43ec73f25b882a87d24749d0ace39b93649065db7ec2e55cc7e49061d16a9d network default NAD default NAD key default], result "{}", err I0428 15:49:33.374068 3817 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:33.374106 3817 pods.go:176] Deleting pod: tsf-tas/ctlog-createtree-job-9pz4t-55l6n I0428 15:49:33.375658 3817 pods.go:236] Attempting to release IPs for pod: tsf-tas/ctlog-createtree-job-9pz4t-55l6n, ips: 10.129.0.34 I0428 15:49:33.375703 3817 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:33.831378 3817 cni.go:452] [tsf-tas/rekor-redis-644cd7d779-z6tng 8bc44f2efff79930a85a1ba3da444a4942883ea401e46692576afde3329489b7 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-redis-644cd7d779-z6tng 8bc44f2efff79930a85a1ba3da444a4942883ea401e46692576afde3329489b7 network default NAD default NAD key ] I0428 15:49:33.841794 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-644cd7d779-z6tng, hostIfaceName: 8bc44f2efff7993, network: default, NAD default, SandboxID: "8bc44f2efff79930a85a1ba3da444a4942883ea401e46692576afde3329489b7", PCI device ID: , UID: "5ba13789-1d62-4dec-bb78-efac7a7aad90", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0428 15:49:33.901116 3817 cni.go:473] [tsf-tas/rekor-redis-644cd7d779-z6tng 8bc44f2efff79930a85a1ba3da444a4942883ea401e46692576afde3329489b7 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-redis-644cd7d779-z6tng 8bc44f2efff79930a85a1ba3da444a4942883ea401e46692576afde3329489b7 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"46:7d:ad:e8:98:56\",\"name\":\"8bc44f2efff7993\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e9f0723b-841c-4b2d-b88a-03cffa5d64fd\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:49:34.249106 3817 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:34.249132 3817 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-9pz4t-55l6n of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0428 15:49:36.258998 3817 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0428 15:49:36.948264 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui W0428 15:49:39.514678 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0428 15:49:40.283203 3817 cni.go:452] [tsf/tsf-tekton-configuration-5wjr8 4a2fb3b97735963f898b1eb32f7107361473a24dd95d0bf366c58101b4b3fa28 network default NAD default NAD key ] DEL starting CNI request [tsf/tsf-tekton-configuration-5wjr8 4a2fb3b97735963f898b1eb32f7107361473a24dd95d0bf366c58101b4b3fa28 network default NAD default NAD key ] I0428 15:49:40.376144 3817 cni.go:473] [tsf/tsf-tekton-configuration-5wjr8 4a2fb3b97735963f898b1eb32f7107361473a24dd95d0bf366c58101b4b3fa28 network default NAD default NAD key default] DEL finished CNI request [tsf/tsf-tekton-configuration-5wjr8 4a2fb3b97735963f898b1eb32f7107361473a24dd95d0bf366c58101b4b3fa28 network default NAD default NAD key default], result "{}", err I0428 15:49:40.431621 3817 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:40.431658 3817 pods.go:176] Deleting pod: tsf/tsf-tekton-configuration-5wjr8 I0428 15:49:40.433128 3817 pods.go:236] Attempting to release IPs for pod: tsf/tsf-tekton-configuration-5wjr8, ips: 10.129.0.38 I0428 15:49:40.433159 3817 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:41.454830 3817 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.454873 3817 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-5wjr8 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.486006 3817 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.486032 3817 obj_retry.go:492] Detected object tsf/patch-tekton-config-2rx2d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:41.491294 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-2rx2d I0428 15:49:41.491324 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-2rx2d W0428 15:49:41.527239 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0428 15:49:45.690161 3817 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:45.690749 3817 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:45.708190 3817 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:45.708220 3817 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:45.713419 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0428 15:49:45.713446 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0428 15:49:47.334363 3817 base_network_controller_pods.go:487] [default/tsf-tpa/create-db-txz9b] creating logical port tsf-tpa_create-db-txz9b for pod on switch ip-10-0-0-9.ec2.internal I0428 15:49:47.334463 3817 kube.go:256] Updating pod tsf-tpa/create-db-txz9b I0428 15:49:47.362713 3817 pod.go:62] [tsf-tpa/create-db-txz9b] pod update took 28.272315ms I0428 15:49:47.362745 3817 base_network_controller_pods.go:951] [default/tsf-tpa/create-db-txz9b] addLogicalPort annotation time took 28.314591ms I0428 15:49:47.364017 3817 pods.go:271] [tsf-tpa/create-db-txz9b] addLogicalPort took 29.680439ms, libovsdb time 854.808µs I0428 15:49:47.696244 3817 cni.go:452] [tsf-tpa/create-db-txz9b bb38f4e1c4e20915ec06a98e856a8da5d00cb46abb2cd5b120fe8c53c3084d90 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-db-txz9b bb38f4e1c4e20915ec06a98e856a8da5d00cb46abb2cd5b120fe8c53c3084d90 network default NAD default NAD key ] I0428 15:49:47.721539 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-db-txz9b, hostIfaceName: bb38f4e1c4e2091, network: default, NAD default, SandboxID: "bb38f4e1c4e20915ec06a98e856a8da5d00cb46abb2cd5b120fe8c53c3084d90", PCI device ID: , UID: "dea91914-d4f1-4f8c-9f5f-7d452bed83eb", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0428 15:49:47.774735 3817 cni.go:473] [tsf-tpa/create-db-txz9b bb38f4e1c4e20915ec06a98e856a8da5d00cb46abb2cd5b120fe8c53c3084d90 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-db-txz9b bb38f4e1c4e20915ec06a98e856a8da5d00cb46abb2cd5b120fe8c53c3084d90 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"da:e2:0d:7c:a0:99\",\"name\":\"bb38f4e1c4e2091\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/05fcf35b-44e7-4b13-93cd-6daf9bd1d817\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:49:48.296305 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis I0428 15:49:50.463220 3817 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=6 I0428 15:49:51.349872 3817 cni.go:452] [tsf-tpa/create-db-txz9b bb38f4e1c4e20915ec06a98e856a8da5d00cb46abb2cd5b120fe8c53c3084d90 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-db-txz9b bb38f4e1c4e20915ec06a98e856a8da5d00cb46abb2cd5b120fe8c53c3084d90 network default NAD default NAD key ] I0428 15:49:51.450503 3817 cni.go:473] [tsf-tpa/create-db-txz9b bb38f4e1c4e20915ec06a98e856a8da5d00cb46abb2cd5b120fe8c53c3084d90 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-db-txz9b bb38f4e1c4e20915ec06a98e856a8da5d00cb46abb2cd5b120fe8c53c3084d90 network default NAD default NAD key default], result "{}", err I0428 15:49:51.500705 3817 obj_retry.go:498] Detected object tsf-tpa/create-db-txz9b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:51.500746 3817 pods.go:176] Deleting pod: tsf-tpa/create-db-txz9b I0428 15:49:51.502242 3817 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-db-txz9b, ips: 10.129.0.40 I0428 15:49:51.502275 3817 obj_retry.go:498] Detected object tsf-tpa/create-db-txz9b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:52.525477 3817 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:52.525509 3817 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:57.190206 3817 obj_retry.go:498] Detected object tsf-tpa/migrate-db-cfwnr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:57.190836 3817 obj_retry.go:498] Detected object tsf-tpa/migrate-db-cfwnr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:49:58.214011 3817 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:58.214039 3817 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:49:59.785101 3817 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-4sjvv-w8b87] creating logical port tsf-tas_tuf-repository-init-4sjvv-w8b87 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:49:59.785233 3817 kube.go:256] Updating pod tsf-tas/tuf-repository-init-4sjvv-w8b87 I0428 15:49:59.795731 3817 pod.go:62] [tsf-tas/tuf-repository-init-4sjvv-w8b87] pod update took 10.524221ms I0428 15:49:59.795753 3817 base_network_controller_pods.go:951] [default/tsf-tas/tuf-repository-init-4sjvv-w8b87] addLogicalPort annotation time took 10.55017ms I0428 15:49:59.797141 3817 pods.go:271] [tsf-tas/tuf-repository-init-4sjvv-w8b87] addLogicalPort took 12.059276ms, libovsdb time 939.871µs I0428 15:50:00.217699 3817 obj_retry.go:498] Detected object tsf-tpa/create-importers-cgccv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:00.218317 3817 obj_retry.go:498] Detected object tsf-tpa/create-importers-cgccv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:01.247602 3817 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:01.247625 3817 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:01.342846 3817 base_network_controller_pods.go:487] [default/tsf-tpa/server-59c4ddddc-dcrd8] creating logical port tsf-tpa_server-59c4ddddc-dcrd8 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:50:01.342956 3817 kube.go:256] Updating pod tsf-tpa/server-59c4ddddc-dcrd8 I0428 15:50:01.360734 3817 pod.go:62] [tsf-tpa/server-59c4ddddc-dcrd8] pod update took 17.805757ms I0428 15:50:01.360760 3817 base_network_controller_pods.go:951] [default/tsf-tpa/server-59c4ddddc-dcrd8] addLogicalPort annotation time took 17.838301ms I0428 15:50:01.362122 3817 pods.go:271] [tsf-tpa/server-59c4ddddc-dcrd8] addLogicalPort took 19.296248ms, libovsdb time 963.733µs I0428 15:50:01.379306 3817 base_network_controller_pods.go:487] [default/tsf-tpa/importer-86bcdf477b-2f8h6] creating logical port tsf-tpa_importer-86bcdf477b-2f8h6 for pod on switch ip-10-0-0-9.ec2.internal I0428 15:50:01.379402 3817 kube.go:256] Updating pod tsf-tpa/importer-86bcdf477b-2f8h6 I0428 15:50:01.398117 3817 pod.go:62] [tsf-tpa/importer-86bcdf477b-2f8h6] pod update took 18.742162ms I0428 15:50:01.398144 3817 base_network_controller_pods.go:951] [default/tsf-tpa/importer-86bcdf477b-2f8h6] addLogicalPort annotation time took 18.775616ms I0428 15:50:01.399497 3817 pods.go:271] [tsf-tpa/importer-86bcdf477b-2f8h6] addLogicalPort took 20.209999ms, libovsdb time 826.519µs I0428 15:50:04.171253 3817 cni.go:452] [tsf-tas/tuf-repository-init-4sjvv-w8b87 28f5dba6398194c74a8c90aba679cf95fcfe48fe4bf9ba77e9f631e0449ad1f5 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-repository-init-4sjvv-w8b87 28f5dba6398194c74a8c90aba679cf95fcfe48fe4bf9ba77e9f631e0449ad1f5 network default NAD default NAD key ] I0428 15:50:04.180656 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-4sjvv-w8b87, hostIfaceName: 28f5dba6398194c, network: default, NAD default, SandboxID: "28f5dba6398194c74a8c90aba679cf95fcfe48fe4bf9ba77e9f631e0449ad1f5", PCI device ID: , UID: "66a934fa-cc13-4ce3-bdf2-872c565b38b8", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0428 15:50:04.234213 3817 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:04.234780 3817 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:04.235582 3817 cni.go:473] [tsf-tas/tuf-repository-init-4sjvv-w8b87 28f5dba6398194c74a8c90aba679cf95fcfe48fe4bf9ba77e9f631e0449ad1f5 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-repository-init-4sjvv-w8b87 28f5dba6398194c74a8c90aba679cf95fcfe48fe4bf9ba77e9f631e0449ad1f5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:8a:b9:ef:61:0d\",\"name\":\"28f5dba6398194c\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d9bec775-4a2a-4d2e-8418-21352d1425fb\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:50:05.254693 3817 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:05.254757 3817 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623190-qm4qv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:07.146893 3817 cni.go:452] [tsf-tpa/importer-86bcdf477b-2f8h6 bfe6e4c74fb8e44b5c8cbe4041b32c3dd659250d8c5d439ae7ae0f616b39e86b network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/importer-86bcdf477b-2f8h6 bfe6e4c74fb8e44b5c8cbe4041b32c3dd659250d8c5d439ae7ae0f616b39e86b network default NAD default NAD key ] I0428 15:50:07.158218 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: importer-86bcdf477b-2f8h6, hostIfaceName: bfe6e4c74fb8e44, network: default, NAD default, SandboxID: "bfe6e4c74fb8e44b5c8cbe4041b32c3dd659250d8c5d439ae7ae0f616b39e86b", PCI device ID: , UID: "fb29bc16-872b-4d40-8d39-1c41d1bc6e05", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0428 15:50:07.217723 3817 cni.go:473] [tsf-tpa/importer-86bcdf477b-2f8h6 bfe6e4c74fb8e44b5c8cbe4041b32c3dd659250d8c5d439ae7ae0f616b39e86b network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/importer-86bcdf477b-2f8h6 bfe6e4c74fb8e44b5c8cbe4041b32c3dd659250d8c5d439ae7ae0f616b39e86b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:ea:11:2c:be:18\",\"name\":\"bfe6e4c74fb8e44\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fc58fb25-b020-4722-ac4a-19f8019300e6\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0428 15:50:07.409859 3817 cni.go:452] [tsf-tpa/server-59c4ddddc-dcrd8 6f791772c04767a6301ff3a1182ef1c85211a3f589d09ee93e9a9e15057d32b3 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/server-59c4ddddc-dcrd8 6f791772c04767a6301ff3a1182ef1c85211a3f589d09ee93e9a9e15057d32b3 network default NAD default NAD key ] I0428 15:50:07.419708 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: server-59c4ddddc-dcrd8, hostIfaceName: 6f791772c04767a, network: default, NAD default, SandboxID: "6f791772c04767a6301ff3a1182ef1c85211a3f589d09ee93e9a9e15057d32b3", PCI device ID: , UID: "f143fccd-8ff0-4ffe-89f3-faed0527367d", MAC: 0a:58:0a:81:00:2a, IPs: [10.129.0.42/23] I0428 15:50:07.477161 3817 cni.go:473] [tsf-tpa/server-59c4ddddc-dcrd8 6f791772c04767a6301ff3a1182ef1c85211a3f589d09ee93e9a9e15057d32b3 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/server-59c4ddddc-dcrd8 6f791772c04767a6301ff3a1182ef1c85211a3f589d09ee93e9a9e15057d32b3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:58:40:9f:dc:d5\",\"name\":\"6f791772c04767a\"},{\"mac\":\"0a:58:0a:81:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d04b392c-fd95-4142-bf78-40aee9ffa11b\"}],\"ips\":[{\"address\":\"10.129.0.42/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:50:08.447768 3817 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0428 15:50:09.454458 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service tsf-tpa/server I0428 15:50:11.468151 3817 cni.go:452] [tsf-tas/tuf-repository-init-4sjvv-w8b87 28f5dba6398194c74a8c90aba679cf95fcfe48fe4bf9ba77e9f631e0449ad1f5 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/tuf-repository-init-4sjvv-w8b87 28f5dba6398194c74a8c90aba679cf95fcfe48fe4bf9ba77e9f631e0449ad1f5 network default NAD default NAD key ] I0428 15:50:11.552324 3817 cni.go:473] [tsf-tas/tuf-repository-init-4sjvv-w8b87 28f5dba6398194c74a8c90aba679cf95fcfe48fe4bf9ba77e9f631e0449ad1f5 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/tuf-repository-init-4sjvv-w8b87 28f5dba6398194c74a8c90aba679cf95fcfe48fe4bf9ba77e9f631e0449ad1f5 network default NAD default NAD key default], result "{}", err I0428 15:50:11.608801 3817 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:11.608841 3817 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-4sjvv-w8b87 I0428 15:50:11.610162 3817 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-4sjvv-w8b87, ips: 10.129.0.41 I0428 15:50:11.610191 3817 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:12.639224 3817 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:12.639251 3817 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-4sjvv-w8b87 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:16.907791 3817 base_network_controller_pods.go:487] [default/tsf-tas/tuf-97fc8ff76-74psc] creating logical port tsf-tas_tuf-97fc8ff76-74psc for pod on switch ip-10-0-0-9.ec2.internal I0428 15:50:16.907892 3817 kube.go:256] Updating pod tsf-tas/tuf-97fc8ff76-74psc I0428 15:50:16.924706 3817 pod.go:62] [tsf-tas/tuf-97fc8ff76-74psc] pod update took 16.837796ms I0428 15:50:16.924736 3817 base_network_controller_pods.go:951] [default/tsf-tas/tuf-97fc8ff76-74psc] addLogicalPort annotation time took 16.873925ms I0428 15:50:16.926081 3817 pods.go:271] [tsf-tas/tuf-97fc8ff76-74psc] addLogicalPort took 18.309582ms, libovsdb time 929.551µs I0428 15:50:22.073638 3817 cni.go:452] [tsf-tas/tuf-97fc8ff76-74psc 0ab3e314e274b70ff6514ce22c7c9c9a088d175db879c5c25892eaf0eae974a4 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-97fc8ff76-74psc 0ab3e314e274b70ff6514ce22c7c9c9a088d175db879c5c25892eaf0eae974a4 network default NAD default NAD key ] I0428 15:50:22.082855 3817 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-97fc8ff76-74psc, hostIfaceName: 0ab3e314e274b70, network: default, NAD default, SandboxID: "0ab3e314e274b70ff6514ce22c7c9c9a088d175db879c5c25892eaf0eae974a4", PCI device ID: , UID: "153b41ad-3ad6-4218-a193-1672efe724cf", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0428 15:50:22.135579 3817 cni.go:473] [tsf-tas/tuf-97fc8ff76-74psc 0ab3e314e274b70ff6514ce22c7c9c9a088d175db879c5c25892eaf0eae974a4 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-97fc8ff76-74psc 0ab3e314e274b70ff6514ce22c7c9c9a088d175db879c5c25892eaf0eae974a4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:b7:6c:68:6f:28\",\"name\":\"0ab3e314e274b70\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b7741b0-40a1-4d65-ab77-b80144f1e71c\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0428 15:50:28.561523 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf I0428 15:50:29.956292 3817 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:29.957013 3817 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:29.994918 3817 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:29.994947 3817 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:50:30.000155 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0428 15:50:30.000178 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0428 15:50:30.710545 3817 namespace.go:98] [image-controller] adding namespace I0428 15:50:30.711412 3817 namespace.go:102] [image-controller] adding namespace took 827.689µs I0428 15:50:30.718151 3817 namespace.go:98] [build-service] adding namespace I0428 15:50:30.718249 3817 namespace.go:98] [integration-service] adding namespace I0428 15:50:30.718884 3817 namespace.go:102] [build-service] adding namespace took 717.961µs I0428 15:50:30.719546 3817 namespace.go:102] [integration-service] adding namespace took 1.280275ms I0428 15:50:30.734155 3817 namespace.go:98] [default-tenant] adding namespace I0428 15:50:30.734955 3817 namespace.go:102] [default-tenant] adding namespace took 782.485µs I0428 15:50:30.765916 3817 namespace.go:142] [image-controller] updating namespace I0428 15:50:30.800799 3817 namespace.go:142] [image-controller] updating namespace I0428 15:50:30.811449 3817 namespace.go:142] [build-service] updating namespace I0428 15:50:30.833602 3817 namespace.go:142] [build-service] updating namespace I0428 15:50:30.843637 3817 namespace.go:142] [integration-service] updating namespace I0428 15:50:30.865780 3817 namespace.go:142] [integration-service] updating namespace I0428 15:50:30.876237 3817 namespace.go:142] [default-tenant] updating namespace I0428 15:50:30.895254 3817 namespace.go:142] [default-tenant] updating namespace I0428 15:50:31.003802 3817 namespace.go:142] [build-service] updating namespace I0428 15:50:31.040097 3817 namespace.go:142] [integration-service] updating namespace I0428 15:50:31.076826 3817 namespace.go:98] [release-service] adding namespace I0428 15:50:31.077676 3817 namespace.go:102] [release-service] adding namespace took 825.949µs I0428 15:50:31.136721 3817 namespace.go:142] [konflux-ui] updating namespace I0428 15:50:31.168654 3817 namespace.go:142] [release-service] updating namespace I0428 15:50:31.205881 3817 namespace.go:142] [release-service] updating namespace I0428 15:50:31.294732 3817 namespace.go:98] [konflux-info] adding namespace I0428 15:50:31.295615 3817 namespace.go:102] [konflux-info] adding namespace took 860.394µs I0428 15:50:31.406662 3817 namespace.go:142] [konflux-info] updating namespace I0428 15:50:31.443112 3817 namespace.go:98] [namespace-lister] adding namespace I0428 15:50:31.444139 3817 namespace.go:102] [namespace-lister] adding namespace took 1.004635ms I0428 15:50:31.456256 3817 namespace.go:142] [konflux-info] updating namespace I0428 15:50:31.533633 3817 namespace.go:98] [enterprise-contract-service] adding namespace I0428 15:50:31.533770 3817 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:31.534478 3817 namespace.go:102] [enterprise-contract-service] adding namespace took 822.896µs I0428 15:50:31.579599 3817 namespace.go:142] [image-controller] updating namespace I0428 15:50:31.665007 3817 namespace.go:142] [default-tenant] updating namespace I0428 15:50:31.858302 3817 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:31.999538 3817 namespace.go:98] [segment-bridge] adding namespace I0428 15:50:32.000421 3817 namespace.go:102] [segment-bridge] adding namespace took 862.279µs I0428 15:50:32.138599 3817 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:32.138626 3817 namespace.go:98] [konflux-cli] adding namespace I0428 15:50:32.139475 3817 namespace.go:102] [konflux-cli] adding namespace took 824.917µs I0428 15:50:32.320209 3817 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:32.331540 3817 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0428 15:50:32.332183 3817 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0428 15:50:32.332203 3817 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0428 15:50:32.334875 3817 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0428 15:50:32.443002 3817 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0428 15:50:32.443028 3817 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0428 15:50:32.443927 3817 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0428 15:50:32.487716 3817 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:32.640723 3817 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:32.942686 3817 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:32.968752 3817 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:33.418536 3817 namespace.go:142] [build-service] updating namespace I0428 15:50:33.755653 3817 namespace.go:142] [default-tenant] updating namespace I0428 15:50:34.084650 3817 namespace.go:142] [image-controller] updating namespace I0428 15:50:34.341982 3817 namespace.go:142] [integration-service] updating namespace I0428 15:50:34.552909 3817 namespace.go:142] [release-service] updating namespace I0428 15:50:34.845067 3817 namespace.go:142] [build-service] updating namespace I0428 15:50:34.948903 3817 namespace.go:142] [default-tenant] updating namespace I0428 15:50:35.088614 3817 namespace.go:142] [image-controller] updating namespace I0428 15:50:35.185995 3817 namespace.go:142] [integration-service] updating namespace I0428 15:50:35.249675 3817 namespace.go:142] [release-service] updating namespace I0428 15:50:35.430429 3817 namespace.go:142] [build-service] updating namespace I0428 15:50:35.726344 3817 namespace.go:142] [default-tenant] updating namespace I0428 15:50:35.839681 3817 namespace.go:142] [image-controller] updating namespace I0428 15:50:35.920909 3817 namespace.go:142] [integration-service] updating namespace I0428 15:50:35.985111 3817 namespace.go:142] [release-service] updating namespace W0428 15:50:36.877673 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0428 15:50:37.238211 3817 namespace.go:142] [enterprise-contract-service] updating namespace W0428 15:50:37.295498 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0428 15:50:37.437704 3817 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:37.635695 3817 namespace.go:142] [konflux-info] updating namespace I0428 15:50:37.838998 3817 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:38.037833 3817 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:38.158490 3817 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:38.172462 3817 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:38.188784 3817 namespace.go:142] [konflux-info] updating namespace I0428 15:50:38.205342 3817 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:38.222577 3817 namespace.go:142] [segment-bridge] updating namespace I0428 15:50:38.268216 3817 namespace.go:142] [enterprise-contract-service] updating namespace I0428 15:50:38.371626 3817 namespace.go:142] [konflux-cli] updating namespace I0428 15:50:38.464629 3817 namespace.go:142] [konflux-info] updating namespace I0428 15:50:38.566515 3817 namespace.go:142] [namespace-lister] updating namespace I0428 15:50:38.662958 3817 namespace.go:142] [segment-bridge] updating namespace W0428 15:50:39.583293 3817 util.go:959] Failed to build node endpoints for node ip-10-0-0-9.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf W0428 15:50:41.353731 3817 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0428 15:50:42.464154 3817 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 W0428 15:50:43.337245 3817 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0428 15:50:47.927073 3817 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:47.927092 3817 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:48.387497 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0428 15:50:48.388050 3817 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 15:50:48.454029 3817 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:48.454048 3817 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0428 15:50:48.474211 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0428 15:50:48.476163 3817 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0428 15:50:48.498319 3817 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0428 15:50:48.498388 3817 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service I0428 15:50:59.753500 3817 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:50:59.754124 3817 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:51:00.454926 3817 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:00.454950 3817 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:00.460837 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p I0428 15:51:00.460868 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-5qf9p I0428 15:51:07.479929 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=7 I0428 15:51:57.464396 3817 obj_retry.go:498] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:51:57.465025 3817 obj_retry.go:498] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:51:57.765107 3817 obj_retry.go:492] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:57.765135 3817 obj_retry.go:492] Detected object konflux-ui/proxy-c8b546c9d-qbnsp of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:51:57.770455 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-c8b546c9d-qbnsp I0428 15:51:57.770481 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-c8b546c9d-qbnsp I0428 15:52:02.938655 3817 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 15:52:02.939285 3817 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 15:52:02.955065 3817 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:52:02.955083 3817 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:52:02.964905 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0428 15:52:02.964938 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0428 15:52:19.489522 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0428 15:52:22.390166 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=154 I0428 15:52:32.870293 3817 namespace.go:98] [managed-tenant-v8l0n] adding namespace I0428 15:52:32.871270 3817 namespace.go:102] [managed-tenant-v8l0n] adding namespace took 954.687µs I0428 15:52:32.893035 3817 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:32.911021 3817 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:34.056008 3817 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:34.193297 3817 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:34.295507 3817 namespace.go:142] [managed-tenant-v8l0n] updating namespace I0428 15:52:37.376622 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=53 I0428 15:52:51.378075 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=172 I0428 15:52:56.378856 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0428 15:53:05.495582 3817 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0428 15:53:05.500463 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0428 15:54:07.380192 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=63 I0428 15:54:38.468511 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=8 I0428 15:54:43.476206 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 I0428 15:56:05.387073 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=488 I0428 15:57:05.488596 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0428 15:57:22.500810 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0428 15:57:32.466727 3817 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0428 15:58:14.464978 3817 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0428 15:58:38.379147 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=25 I0428 15:59:31.481861 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0428 15:59:49.383264 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=7 I0428 15:59:50.381219 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0428 15:59:52.045249 3817 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:52.045305 3817 obj_retry.go:492] Detected object tsf-tpa/create-db-txz9b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:52.049295 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-txz9b I0428 15:59:52.049317 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-txz9b I0428 15:59:58.027327 3817 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:58.027380 3817 obj_retry.go:492] Detected object tsf-tpa/migrate-db-cfwnr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 15:59:58.030173 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-cfwnr I0428 15:59:58.030209 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-cfwnr I0428 16:00:01.027087 3817 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:01.027147 3817 obj_retry.go:492] Detected object tsf-tpa/create-importers-cgccv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:01.029418 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-cgccv I0428 16:00:01.029448 3817 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-cgccv I0428 16:00:04.512064 3817 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0428 16:00:04.512625 3817 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0428 16:00:05.538287 3817 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:05.538318 3817 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29623200-9j7ns of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0428 16:00:14.470711 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=5 I0428 16:00:17.492362 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=9 I0428 16:00:24.478645 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0428 16:00:51.502822 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0428 16:01:24.392769 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=8 I0428 16:01:40.498234 3817 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0428 16:01:46.389506 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=31 I0428 16:02:17.380384 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=10 I0428 16:02:48.490779 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0428 16:03:57.469407 3817 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=6 I0428 16:05:11.381179 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=29 I0428 16:05:33.383413 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0428 16:06:00.503936 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=9 I0428 16:06:26.395030 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=5 I0428 16:06:57.484451 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 I0428 16:07:06.466880 3817 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0428 16:07:07.473218 3817 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0428 16:07:26.385139 3817 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8