+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-0-38.ec2.internal ++ [[ -n ip-10-0-0-38.ec2.internal ]] ++ [[ -f /env/ip-10-0-0-38.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0429 08:38:43.667266936 - disable conntrack on geneve port' I0429 08:38:43.667266936 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0429 08:38:43.702361494 - starting ovnkube-node + echo 'I0429 08:38:43.702361494 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-0-38.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-0-38.ec2.internal --init-node ip-10-0-0-38.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-0-38.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0429 08:38:43.745444 3791 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0429 08:38:43.745499 3791 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-dc35fd4ddb.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0429 08:38:43.746963 3791 kube.go:431] Waiting for certificate I0429 08:38:43.746989 3791 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0429 08:38:43.747382 3791 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0429 08:38:43.748145 3791 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0429 08:38:43.786159 3791 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 08:38:43.786174 3791 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 08:38:43.791599 3791 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 08:38:43.791702 3791 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-x2vgx" I0429 08:38:43.796761 3791 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-x2vgx" I0429 08:38:43.796854 3791 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 08:38:44.747696 3791 kube.go:438] Certificate found I0429 08:38:44.748221 3791 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0429 08:38:44.749211 3791 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0429 08:38:44.813207 3791 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-30 08:33:43 +0000 UTC" deadline="2026-04-30 01:42:56.497170918 +0000 UTC" I0429 08:38:44.813253 3791 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="17h4m11.683932374s" I0429 08:38:44.817255 3791 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0429 08:38:44.839601 3791 controller_manager.go:370] Starting the ovnkube controller I0429 08:38:44.839621 3791 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-0-38.ec2.internal I0429 08:38:44.839688 3791 controller_manager.go:395] NBDB zone sync took: 53.846µs I0429 08:38:44.839704 3791 factory.go:561] Starting watch factory I0429 08:38:44.839845 3791 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.839870 3791 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.839893 3791 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.839880 3791 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.839894 3791 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.839911 3791 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.840165 3791 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.840197 3791 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.840231 3791 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.840250 3791 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.840479 3791 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.840577 3791 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.842052 3791 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0429 08:38:44.842139 3791 factory.go:561] Starting watch factory I0429 08:38:44.842858 3791 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0429 08:38:44.842996 3791 server.go:101] MetricServer registers OVS metrics I0429 08:38:44.843284 3791 server.go:105] MetricServer registers OVN DB metrics I0429 08:38:44.848026 3791 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0429 08:38:44.848049 3791 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0429 08:38:44.855332 3791 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.855455 3791 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.855547 3791 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.857597 3791 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.857976 3791 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.862951 3791 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0429 08:38:44.864504 3791 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0429 08:38:44.864603 3791 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0429 08:38:44.868856 3791 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0429 08:38:44.868909 3791 server.go:109] MetricServer registers OVN Controller metrics I0429 08:38:44.873417 3791 server.go:113] MetricServer registers OVN Northd metrics I0429 08:38:44.877179 3791 metrics.go:519] OVN Metrics Server starts to run ... I0429 08:38:44.945023 3791 factory.go:1977] *v1.Namespace informer cache synced successfully I0429 08:38:44.945041 3791 factory.go:1977] *v1.Node informer cache synced successfully I0429 08:38:44.945045 3791 factory.go:1977] *v1.Service informer cache synced successfully I0429 08:38:44.945049 3791 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0429 08:38:44.945052 3791 factory.go:1977] *v1.Pod informer cache synced successfully I0429 08:38:44.945055 3791 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0429 08:38:44.945140 3791 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 08:38:44.945149 3791 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 08:38:44.945163 3791 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 08:38:44.945153 3791 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 08:38:44.947574 3791 factory.go:1977] *v1.Namespace informer cache synced successfully I0429 08:38:44.947587 3791 factory.go:1977] *v1.Node informer cache synced successfully I0429 08:38:44.947590 3791 factory.go:1977] *v1.Service informer cache synced successfully I0429 08:38:44.947593 3791 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0429 08:38:44.947597 3791 factory.go:1977] *v1.Pod informer cache synced successfully I0429 08:38:44.947600 3791 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0429 08:38:44.947985 3791 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 08:38:44.947983 3791 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 08:38:44.950546 3791 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0429 08:38:44.950556 3791 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0429 08:38:44.950619 3791 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.950636 3791 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.952666 3791 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0429 08:38:44.952676 3791 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0429 08:38:44.953086 3791 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.955609 3791 factory.go:1977] *v1.EgressIP informer cache synced successfully I0429 08:38:44.955659 3791 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.955670 3791 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.957726 3791 factory.go:1977] *v1.EgressIP informer cache synced successfully I0429 08:38:44.958253 3791 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.960789 3791 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0429 08:38:44.960849 3791 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.960859 3791 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.962933 3791 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0429 08:38:44.963201 3791 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.966150 3791 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0429 08:38:44.966202 3791 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.966215 3791 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.968145 3791 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0429 08:38:44.968373 3791 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.971245 3791 factory.go:1977] *v1.EgressService informer cache synced successfully I0429 08:38:44.971294 3791 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.971304 3791 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.973277 3791 factory.go:1977] *v1.EgressService informer cache synced successfully I0429 08:38:44.973303 3791 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.977250 3791 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0429 08:38:44.977412 3791 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0429 08:38:44.977431 3791 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0429 08:38:44.978630 3791 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0429 08:38:44.979716 3791 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0429 08:38:44.982550 3791 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0429 08:38:44.982608 3791 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.982620 3791 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.982650 3791 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.982662 3791 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.983841 3791 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0429 08:38:44.984912 3791 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.984929 3791 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 08:38:44.988016 3791 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0429 08:38:44.988027 3791 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0429 08:38:44.988034 3791 factory.go:680] Watch Factory start up complete, took: 148.333101ms I0429 08:38:44.988040 3791 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-0-38.ec2.internal" zone I0429 08:38:44.989156 3791 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0429 08:38:44.989165 3791 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0429 08:38:44.989170 3791 factory.go:680] Watch Factory start up complete, took: 147.03282ms I0429 08:38:44.989615 3791 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0429 08:38:45.037217 3791 default_node_network_controller.go:710] Initializing the default node network controller I0429 08:38:45.124098 3791 udn_isolation.go:101] Starting UDN host isolation manager I0429 08:38:45.130300 3791 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0429 08:38:45.187655 3791 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0429 08:38:45.187728 3791 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0429 08:38:45.187745 3791 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0429 08:38:45.219696 3791 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0429 08:38:45.219759 3791 default_node_network_controller.go:809] Node ip-10-0-0-38.ec2.internal ready for ovn initialization with subnet 10.128.0.0/23 I0429 08:38:45.225261 3791 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.0.38"] k8s.ovn.org/zone-name:ip-10-0-0-38.ec2.internal] on node ip-10-0-0-38.ec2.internal I0429 08:38:45.249439 3791 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0429 08:38:45.255177 3791 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0429 08:38:45.255308 3791 helper_linux.go:118] Found default gateway interface br-ex 10.0.0.1 I0429 08:38:45.255482 3791 gateway_init.go:263] Preparing Gateway I0429 08:38:45.255493 3791 gateway_shared_intf.go:1687] Creating new gateway I0429 08:38:45.362796 3791 iptables.go:108] Creating table: filter chain: FORWARD I0429 08:38:45.368795 3791 iptables.go:108] Creating table: filter chain: OUTPUT I0429 08:38:45.374565 3791 gateway_shared_intf.go:1816] Gateway Creation Complete I0429 08:38:45.374578 3791 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-38.ec2.internal","mac-address":"12:64:c3:8c:ba:09","ip-addresses":["10.0.0.38/24"],"ip-address":"10.0.0.38/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:206d94dc-98da-4ade-8301-6a072e04efe5 k8s.ovn.org/node-encap-ips:["10.0.0.38"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.38/24"} k8s.ovn.org/zone-name:ip-10-0-0-38.ec2.internal] on node ip-10-0-0-38.ec2.internal I0429 08:38:45.394796 3791 default_node_network_controller.go:519] Node connection status = connected I0429 08:38:45.488891 3791 controller_manager.go:429] Waiting for node in zone sync took: 500.839252ms I0429 08:38:45.495929 3791 controller_manager.go:611] Switching to transit router for layer2 networks I0429 08:38:45.495943 3791 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-0-38.ec2.internal I0429 08:38:45.514376 3791 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0429 08:38:45.520792 3791 controller_manager.go:290] SCTP support detected in OVN I0429 08:38:45.521342 3791 services_controller.go:69] Creating services controller for network=default I0429 08:38:45.521498 3791 udn_enabled_svc.go:80] Waiting for service informer to sync I0429 08:38:45.521551 3791 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0429 08:38:45.521638 3791 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0429 08:38:45.521652 3791 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0429 08:38:45.521661 3791 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0429 08:38:45.521912 3791 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0429 08:38:45.521955 3791 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0429 08:38:45.522040 3791 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 88.783µs I0429 08:38:45.522064 3791 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 102.227µs I0429 08:38:45.522072 3791 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 117.403µs I0429 08:38:45.522078 3791 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0429 08:38:45.522083 3791 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0429 08:38:45.522084 3791 udn_enabled_svc.go:95] Performing full resync I0429 08:38:45.522091 3791 default_network_controller.go:325] Starting the default network controller I0429 08:38:45.522191 3791 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 13.008µs I0429 08:38:45.522419 3791 udn_enabled_svc.go:99] Waiting for handler to sync I0429 08:38:45.522430 3791 udn_enabled_svc.go:104] Starting worker I0429 08:38:45.531034 3791 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0429 08:38:45.531084 3791 repair.go:33] Syncing exgw routes took 39.191µs I0429 08:38:45.531104 3791 default_network_controller.go:414] Starting all the Watchers... I0429 08:38:45.531238 3791 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0429 08:38:45.531241 3791 namespace.go:98] [openshift-host-network] adding namespace I0429 08:38:45.531250 3791 namespace.go:98] [openshift-backplane-cee] adding namespace I0429 08:38:45.531254 3791 namespace.go:98] [openshift-config] adding namespace I0429 08:38:45.531267 3791 namespace.go:98] [openshift] adding namespace I0429 08:38:45.531266 3791 namespace.go:98] [openshift-etcd] adding namespace I0429 08:38:45.531274 3791 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0429 08:38:45.531275 3791 namespace.go:98] [openshift-package-operator] adding namespace I0429 08:38:45.531283 3791 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0429 08:38:45.531293 3791 namespace.go:98] [openshift-console-user-settings] adding namespace I0429 08:38:45.531303 3791 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0429 08:38:45.531307 3791 namespace.go:98] [kube-public] adding namespace I0429 08:38:45.531312 3791 namespace.go:98] [kube-system] adding namespace I0429 08:38:45.531316 3791 namespace.go:98] [openshift-dns] adding namespace I0429 08:38:45.531349 3791 namespace.go:98] [openshift-backplane-cse] adding namespace I0429 08:38:45.531817 3791 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 568.692µs I0429 08:38:45.531847 3791 namespace.go:98] [default] adding namespace I0429 08:38:45.532258 3791 namespace.go:102] [default] adding namespace took 405.57µs I0429 08:38:45.532270 3791 namespace.go:98] [openshift-logging] adding namespace I0429 08:38:45.532657 3791 namespace.go:102] [openshift-logging] adding namespace took 373.323µs I0429 08:38:45.532673 3791 namespace.go:98] [openshift-apiserver] adding namespace I0429 08:38:45.533223 3791 namespace.go:102] [openshift-host-network] adding namespace took 1.973966ms I0429 08:38:45.533232 3791 namespace.go:98] [openshift-backplane-tam] adding namespace I0429 08:38:45.533971 3791 namespace.go:102] [openshift-backplane-cee] adding namespace took 2.710897ms I0429 08:38:45.533987 3791 namespace.go:98] [openshift-authentication] adding namespace I0429 08:38:45.534461 3791 namespace.go:102] [openshift-config] adding namespace took 3.198983ms I0429 08:38:45.534470 3791 namespace.go:98] [openshift-kube-scheduler] adding namespace I0429 08:38:45.534864 3791 namespace.go:102] [openshift] adding namespace took 3.591481ms I0429 08:38:45.534878 3791 namespace.go:98] [openshift-network-node-identity] adding namespace I0429 08:38:45.535302 3791 namespace.go:102] [openshift-etcd] adding namespace took 4.026232ms I0429 08:38:45.535316 3791 namespace.go:98] [openshift-machine-config-operator] adding namespace I0429 08:38:45.535739 3791 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 4.458968ms I0429 08:38:45.535753 3791 namespace.go:98] [openshift-operators-redhat] adding namespace I0429 08:38:45.536155 3791 namespace.go:102] [openshift-package-operator] adding namespace took 4.87519ms I0429 08:38:45.536165 3791 namespace.go:98] [openshift-network-operator] adding namespace I0429 08:38:45.536566 3791 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 5.278452ms I0429 08:38:45.536580 3791 namespace.go:98] [openshift-service-ca-operator] adding namespace I0429 08:38:45.536986 3791 namespace.go:102] [openshift-console-user-settings] adding namespace took 5.682677ms I0429 08:38:45.536996 3791 namespace.go:98] [openshift-ingress] adding namespace I0429 08:38:45.537397 3791 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 6.088038ms I0429 08:38:45.537409 3791 namespace.go:98] [kube-node-lease] adding namespace I0429 08:38:45.537790 3791 namespace.go:102] [kube-public] adding namespace took 6.47667ms I0429 08:38:45.537800 3791 namespace.go:98] [openshift-must-gather-operator] adding namespace I0429 08:38:45.538203 3791 namespace.go:102] [kube-system] adding namespace took 6.887333ms I0429 08:38:45.538212 3791 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0429 08:38:45.538591 3791 namespace.go:102] [openshift-dns] adding namespace took 7.272273ms I0429 08:38:45.538600 3791 namespace.go:98] [openshift-config-managed] adding namespace I0429 08:38:45.538992 3791 namespace.go:102] [openshift-backplane-cse] adding namespace took 7.633334ms I0429 08:38:45.539007 3791 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0429 08:38:45.539367 3791 namespace.go:102] [openshift-apiserver] adding namespace took 6.689818ms I0429 08:38:45.539378 3791 namespace.go:98] [openshift-authentication-operator] adding namespace I0429 08:38:45.539721 3791 namespace.go:102] [openshift-backplane-tam] adding namespace took 6.485106ms I0429 08:38:45.539733 3791 namespace.go:98] [dedicated-admin] adding namespace I0429 08:38:45.540120 3791 namespace.go:102] [openshift-authentication] adding namespace took 6.128351ms I0429 08:38:45.540131 3791 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0429 08:38:45.540495 3791 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.020146ms I0429 08:38:45.540509 3791 namespace.go:98] [openshift-node] adding namespace I0429 08:38:45.540857 3791 namespace.go:102] [openshift-network-node-identity] adding namespace took 5.973979ms I0429 08:38:45.540870 3791 namespace.go:98] [openshift-marketplace] adding namespace I0429 08:38:45.541238 3791 namespace.go:102] [openshift-machine-config-operator] adding namespace took 5.917356ms I0429 08:38:45.541250 3791 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0429 08:38:45.541559 3791 namespace.go:102] [openshift-operators-redhat] adding namespace took 5.802144ms I0429 08:38:45.541569 3791 namespace.go:98] [openshift-kube-apiserver] adding namespace I0429 08:38:45.542016 3791 namespace.go:102] [openshift-network-operator] adding namespace took 5.847154ms I0429 08:38:45.542028 3791 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0429 08:38:45.542374 3791 namespace.go:102] [openshift-service-ca-operator] adding namespace took 5.788883ms I0429 08:38:45.542385 3791 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0429 08:38:45.542751 3791 namespace.go:102] [openshift-ingress] adding namespace took 5.751487ms I0429 08:38:45.542763 3791 namespace.go:98] [openshift-infra] adding namespace I0429 08:38:45.543132 3791 namespace.go:102] [kube-node-lease] adding namespace took 5.7171ms I0429 08:38:45.543148 3791 namespace.go:98] [openshift-backplane-mobb] adding namespace I0429 08:38:45.543684 3791 namespace.go:102] [openshift-must-gather-operator] adding namespace took 5.877763ms I0429 08:38:45.543699 3791 namespace.go:98] [openshift-network-console] adding namespace I0429 08:38:45.544072 3791 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 5.853631ms I0429 08:38:45.544086 3791 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0429 08:38:45.544535 3791 namespace.go:102] [openshift-config-managed] adding namespace took 5.930752ms I0429 08:38:45.544546 3791 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0429 08:38:45.544911 3791 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 5.90049ms I0429 08:38:45.544920 3791 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0429 08:38:45.545356 3791 namespace.go:102] [openshift-authentication-operator] adding namespace took 5.972914ms I0429 08:38:45.545367 3791 namespace.go:98] [openshift-insights] adding namespace I0429 08:38:45.545744 3791 namespace.go:102] [dedicated-admin] adding namespace took 6.005238ms I0429 08:38:45.545758 3791 namespace.go:98] [openshift-machine-api] adding namespace I0429 08:38:45.546199 3791 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 6.062198ms I0429 08:38:45.546215 3791 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0429 08:38:45.546686 3791 namespace.go:102] [openshift-node] adding namespace took 6.171116ms I0429 08:38:45.546700 3791 namespace.go:98] [openshift-cluster-version] adding namespace I0429 08:38:45.547144 3791 namespace.go:102] [openshift-marketplace] adding namespace took 6.269537ms I0429 08:38:45.547157 3791 namespace.go:98] [openshift-operators] adding namespace I0429 08:38:45.547590 3791 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 6.335177ms I0429 08:38:45.547602 3791 namespace.go:98] [openshift-console] adding namespace I0429 08:38:45.548040 3791 namespace.go:102] [openshift-kube-apiserver] adding namespace took 6.465521ms I0429 08:38:45.548052 3791 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0429 08:38:45.548488 3791 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 6.45515ms I0429 08:38:45.548501 3791 namespace.go:98] [openshift-dns-operator] adding namespace I0429 08:38:45.548944 3791 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 6.55405ms I0429 08:38:45.548957 3791 namespace.go:98] [openshift-apiserver-operator] adding namespace I0429 08:38:45.549388 3791 namespace.go:102] [openshift-infra] adding namespace took 6.618867ms I0429 08:38:45.549406 3791 namespace.go:98] [openshift-controller-manager-operator] adding namespace I0429 08:38:45.549799 3791 namespace.go:102] [openshift-backplane-mobb] adding namespace took 6.644959ms I0429 08:38:45.549814 3791 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0429 08:38:45.550281 3791 namespace.go:102] [openshift-network-console] adding namespace took 6.57619ms I0429 08:38:45.550297 3791 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0429 08:38:45.550675 3791 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 6.584407ms I0429 08:38:45.550687 3791 namespace.go:98] [openshift-backplane-srep] adding namespace I0429 08:38:45.551169 3791 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 6.617683ms I0429 08:38:45.551182 3791 namespace.go:98] [openshift-config-operator] adding namespace I0429 08:38:45.551576 3791 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 6.649315ms I0429 08:38:45.551592 3791 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0429 08:38:45.552004 3791 namespace.go:102] [openshift-insights] adding namespace took 6.632043ms I0429 08:38:45.552433 3791 namespace.go:102] [openshift-machine-api] adding namespace took 6.669985ms I0429 08:38:45.552445 3791 namespace.go:98] [openshift-network-diagnostics] adding namespace I0429 08:38:45.552839 3791 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 6.604962ms I0429 08:38:45.552852 3791 namespace.go:98] [openshift-multus] adding namespace I0429 08:38:45.553260 3791 namespace.go:102] [openshift-cluster-version] adding namespace took 6.554689ms I0429 08:38:45.553273 3791 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0429 08:38:45.553617 3791 namespace.go:102] [openshift-operators] adding namespace took 6.455619ms I0429 08:38:45.553630 3791 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0429 08:38:45.554129 3791 namespace.go:102] [openshift-console] adding namespace took 6.521531ms I0429 08:38:45.554144 3791 namespace.go:98] [openshift-route-controller-manager] adding namespace I0429 08:38:45.554567 3791 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 6.50888ms I0429 08:38:45.554578 3791 namespace.go:98] [openshift-controller-manager] adding namespace I0429 08:38:45.554989 3791 namespace.go:102] [openshift-dns-operator] adding namespace took 6.483947ms I0429 08:38:45.555000 3791 namespace.go:98] [openshift-backplane] adding namespace I0429 08:38:45.555395 3791 namespace.go:102] [openshift-apiserver-operator] adding namespace took 6.433699ms I0429 08:38:45.555406 3791 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0429 08:38:45.555811 3791 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 6.399888ms I0429 08:38:45.555822 3791 namespace.go:98] [openshift-backplane-csm] adding namespace I0429 08:38:45.556414 3791 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 6.592577ms I0429 08:38:45.556430 3791 namespace.go:98] [openshift-monitoring] adding namespace I0429 08:38:45.556810 3791 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 6.508548ms I0429 08:38:45.556822 3791 namespace.go:98] [open-cluster-management-2pvn461a2e3dg73kigr9ohvaqf1urjn8] adding namespace I0429 08:38:45.557279 3791 namespace.go:102] [openshift-backplane-srep] adding namespace took 6.586166ms I0429 08:38:45.557291 3791 namespace.go:98] [openshift-customer-monitoring] adding namespace I0429 08:38:45.557687 3791 namespace.go:102] [openshift-config-operator] adding namespace took 6.498454ms I0429 08:38:45.557704 3791 namespace.go:98] [openshift-console-operator] adding namespace W0429 08:38:45.558126 3791 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-7t9xt: pod openshift-network-diagnostics/network-check-target-7t9xt: no pod IPs found W0429 08:38:45.558140 3791 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-dp8wx: pod openshift-network-diagnostics/network-check-target-dp8wx: no pod IPs found I0429 08:38:45.558181 3791 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 6.581701ms I0429 08:38:45.558197 3791 namespace.go:98] [openshift-ingress-canary] adding namespace W0429 08:38:45.558770 3791 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-c5n5q: pod openshift-multus/network-metrics-daemon-c5n5q: no pod IPs found W0429 08:38:45.558781 3791 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-fhh6p: pod openshift-multus/network-metrics-daemon-fhh6p: no pod IPs found I0429 08:38:45.558814 3791 namespace.go:102] [openshift-network-diagnostics] adding namespace took 6.364612ms I0429 08:38:45.559719 3791 namespace.go:102] [openshift-multus] adding namespace took 6.861395ms I0429 08:38:45.560158 3791 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 6.877433ms I0429 08:38:45.560635 3791 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 6.999734ms I0429 08:38:45.560649 3791 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0429 08:38:45.561194 3791 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.043087ms I0429 08:38:45.561743 3791 namespace.go:102] [openshift-controller-manager] adding namespace took 7.157213ms I0429 08:38:45.562259 3791 namespace.go:102] [openshift-backplane] adding namespace took 7.253393ms I0429 08:38:45.562796 3791 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.383353ms I0429 08:38:45.562813 3791 namespace.go:98] [openshift-ingress-operator] adding namespace I0429 08:38:45.563253 3791 namespace.go:102] [openshift-backplane-csm] adding namespace took 7.425743ms I0429 08:38:45.563736 3791 namespace.go:102] [openshift-monitoring] adding namespace took 7.300278ms I0429 08:38:45.563752 3791 namespace.go:98] [openshift-image-registry] adding namespace I0429 08:38:45.564191 3791 namespace.go:102] [open-cluster-management-2pvn461a2e3dg73kigr9ohvaqf1urjn8] adding namespace took 7.36284ms I0429 08:38:45.564561 3791 namespace.go:102] [openshift-customer-monitoring] adding namespace took 7.263357ms I0429 08:38:45.565108 3791 namespace.go:102] [openshift-console-operator] adding namespace took 7.397559ms I0429 08:38:45.565607 3791 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.402005ms I0429 08:38:45.566006 3791 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 5.352202ms I0429 08:38:45.566455 3791 namespace.go:102] [openshift-ingress-operator] adding namespace took 3.637045ms I0429 08:38:45.566785 3791 namespace.go:102] [openshift-image-registry] adding namespace took 3.028701ms W0429 08:38:45.566873 3791 master.go:471] Unable to parse remote node ip-10-0-2-176.ec2.internal chassis-id annotation. Chassis may be removed during sync I0429 08:38:45.566969 3791 master.go:548] Adding or Updating local node "ip-10-0-0-38.ec2.internal" for network "default" I0429 08:38:45.567027 3791 obj_retry.go:578] Failed to create *v1.Node ip-10-0-2-176.ec2.internal, error: adding or updating remote node chassis ip-10-0-2-176.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-2-176.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-2-176.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-2-176.ec2.internal W0429 08:38:45.575156 3791 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-fhh6p: pod openshift-multus/network-metrics-daemon-fhh6p: no pod IPs found W0429 08:38:45.575173 3791 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-dp8wx: pod openshift-network-diagnostics/network-check-target-dp8wx: no pod IPs found I0429 08:38:45.576157 3791 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-0-38.ec2.internal for the network default I0429 08:38:45.576562 3791 zone_ic_handler.go:195] Time taken to create transit switch: 391.498µs I0429 08:38:45.577325 3791 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0429 08:38:45.577344 3791 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0429 08:38:45.577357 3791 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0429 08:38:45.577390 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-176.ec2.internal in network "default" I0429 08:38:45.577447 3791 node_tracker.go:235] Node ip-10-0-2-176.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-176.ec2.internal" I0429 08:38:45.577464 3791 node_tracker.go:169] Node ip-10-0-2-176.ec2.internal switch + router changed, syncing services in network "default" I0429 08:38:45.577481 3791 services_controller.go:574] Full service sync requested for network=default I0429 08:38:45.577494 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-38.ec2.internal in network "default" I0429 08:38:45.577532 3791 node_tracker.go:169] Node ip-10-0-0-38.ec2.internal switch + router changed, syncing services in network "default" I0429 08:38:45.577541 3791 services_controller.go:574] Full service sync requested for network=default I0429 08:38:45.677876 3791 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0429 08:38:45.677892 3791 services_controller.go:208] Setting up event handlers for services for network=default I0429 08:38:45.677962 3791 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0429 08:38:45.678012 3791 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0429 08:38:45.678027 3791 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0429 08:38:45.778208 3791 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0429 08:38:45.778223 3791 repair.go:60] Starting repairing loop for services I0429 08:38:45.778309 3791 repair.go:131] Deleted 0 stale service LBs I0429 08:38:45.778328 3791 repair.go:137] Deleted 0 stale Chassis Template Vars I0429 08:38:45.778344 3791 repair.go:62] Finished repairing loop for services: 122.38µs I0429 08:38:45.778366 3791 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0429 08:38:45.778375 3791 services_controller.go:254] Starting workers for network=default I0429 08:38:45.778685 3791 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-dp8wx] creating logical port openshift-network-diagnostics_network-check-target-dp8wx for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:45.778690 3791 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-c5n5q, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-c5n5q: suppressed error logged: pod openshift-multus/network-metrics-daemon-c5n5q: no pod IPs found I0429 08:38:45.778747 3791 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-fhh6p] creating logical port openshift-multus_network-metrics-daemon-fhh6p for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:45.778767 3791 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-7t9xt, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-7t9xt: suppressed error logged: pod openshift-network-diagnostics/network-check-target-7t9xt: no pod IPs found I0429 08:38:45.778896 3791 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-dp8wx I0429 08:38:45.778969 3791 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-fhh6p I0429 08:38:45.797724 3791 pod.go:62] [openshift-multus/network-metrics-daemon-fhh6p] pod update took 18.917703ms I0429 08:38:45.797742 3791 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-fhh6p] addLogicalPort annotation time took 18.937639ms I0429 08:38:45.798513 3791 pods.go:271] [openshift-multus/network-metrics-daemon-fhh6p] addLogicalPort took 19.776514ms, libovsdb time 573.937µs I0429 08:38:45.800963 3791 pod.go:62] [openshift-network-diagnostics/network-check-target-dp8wx] pod update took 22.193957ms I0429 08:38:45.800981 3791 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-dp8wx] addLogicalPort annotation time took 22.214541ms I0429 08:38:45.801714 3791 pods.go:271] [openshift-network-diagnostics/network-check-target-dp8wx] addLogicalPort took 23.042836ms, libovsdb time 550.401µs I0429 08:38:45.802180 3791 admin_network_policy_controller.go:227] Starting controller default-network-controller I0429 08:38:45.802199 3791 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0429 08:38:45.802210 3791 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0429 08:38:45.802214 3791 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0429 08:38:45.802390 3791 repair.go:29] Repairing admin network policies took 171.251µs I0429 08:38:45.802484 3791 repair.go:92] Repairing baseline admin network policies took 85.939µs I0429 08:38:45.802906 3791 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0429 08:38:45.802921 3791 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0429 08:38:45.804259 3791 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0429 08:38:45.804285 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0429 08:38:45.804295 3791 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0429 08:38:45.805929 3791 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0429 08:38:45.805946 3791 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0429 08:38:45.805954 3791 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0429 08:38:45.807611 3791 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0429 08:38:45.807628 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0429 08:38:45.807636 3791 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0429 08:38:45.808134 3791 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0429 08:38:45.808152 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0429 08:38:45.808161 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0429 08:38:45.809649 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0429 08:38:45.809666 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0429 08:38:45.809674 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0429 08:38:45.810181 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0429 08:38:45.810199 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0429 08:38:45.810208 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0429 08:38:45.811613 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0429 08:38:45.811630 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0429 08:38:45.812004 3791 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0429 08:38:45.812018 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0429 08:38:45.813129 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0429 08:38:45.813145 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0429 08:38:45.813155 3791 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0429 08:38:45.813611 3791 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0429 08:38:45.813630 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0429 08:38:45.813638 3791 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0429 08:38:45.814102 3791 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0429 08:38:45.814118 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0429 08:38:45.814124 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0429 08:38:45.814686 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0429 08:38:45.814701 3791 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0429 08:38:45.814710 3791 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0429 08:38:45.816209 3791 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0429 08:38:45.816224 3791 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0429 08:38:45.816232 3791 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0429 08:38:45.817110 3791 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0429 08:38:45.817129 3791 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0429 08:38:45.817139 3791 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0429 08:38:45.819293 3791 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0429 08:38:45.819311 3791 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0429 08:38:45.819321 3791 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0429 08:38:45.820253 3791 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0429 08:38:45.820282 3791 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0429 08:38:45.820294 3791 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0429 08:38:45.821515 3791 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0429 08:38:45.821533 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0429 08:38:45.821541 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0429 08:38:45.822050 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0429 08:38:45.822068 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0429 08:38:45.822089 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0429 08:38:45.822755 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0429 08:38:45.822773 3791 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0429 08:38:45.822781 3791 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0429 08:38:45.823816 3791 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0429 08:38:45.823851 3791 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0429 08:38:45.823858 3791 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0429 08:38:45.824167 3791 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0429 08:38:45.824184 3791 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0429 08:38:45.824192 3791 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0429 08:38:45.825305 3791 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0429 08:38:45.825321 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0429 08:38:45.825327 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0429 08:38:45.825613 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0429 08:38:45.825631 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0429 08:38:45.825639 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0429 08:38:45.826889 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0429 08:38:45.826906 3791 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0429 08:38:45.826915 3791 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0429 08:38:45.827976 3791 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0429 08:38:45.827992 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0429 08:38:45.827998 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0429 08:38:45.828457 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0429 08:38:45.828476 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0429 08:38:45.828482 3791 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0429 08:38:45.828739 3791 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0429 08:38:45.828754 3791 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0429 08:38:45.828761 3791 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0429 08:38:45.829235 3791 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0429 08:38:45.829253 3791 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0429 08:38:45.829260 3791 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0429 08:38:45.829701 3791 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0429 08:38:45.829716 3791 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0429 08:38:45.829726 3791 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0429 08:38:45.830596 3791 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0429 08:38:45.834927 3791 egressfirewall.go:374] Starting EgressFirewall controller I0429 08:38:45.834963 3791 controller.go:133] Adding controller egress-firewall-controller event handlers I0429 08:38:45.834974 3791 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0429 08:38:45.835010 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0429 08:38:45.835026 3791 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0429 08:38:45.835049 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0429 08:38:45.835061 3791 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0429 08:38:45.835347 3791 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0429 08:38:45.835360 3791 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0429 08:38:45.835366 3791 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0429 08:38:45.835374 3791 egressqos.go:193] Setting up event handlers for EgressQoS I0429 08:38:45.835382 3791 egressfirewall.go:723] Syncing node "ip-10-0-0-38.ec2.internal" for egress firewall I0429 08:38:45.835390 3791 egressfirewall.go:723] Syncing node "ip-10-0-2-176.ec2.internal" for egress firewall I0429 08:38:45.835578 3791 egressqos.go:245] Starting EgressQoS Controller I0429 08:38:45.835632 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0429 08:38:45.835647 3791 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0429 08:38:45.835655 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0429 08:38:45.835661 3791 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0429 08:38:45.835669 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0429 08:38:45.835675 3791 shared_informer.go:356] "Caches are synced" controller="egressqos" I0429 08:38:45.835679 3791 egressqos.go:259] Repairing EgressQoSes I0429 08:38:45.835683 3791 egressqos.go:399] Starting repairing loop for egressqos I0429 08:38:45.835762 3791 egressqos.go:401] Finished repairing loop for egressqos: 78.119µs I0429 08:38:45.835776 3791 egressservice_zone.go:132] Setting up event handlers for Egress Services I0429 08:38:45.835810 3791 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-0-38.ec2.internal I0429 08:38:45.835880 3791 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-0-38.ec2.internal : 69.992µs I0429 08:38:45.836118 3791 egressservice_zone.go:210] Starting Egress Services Controller I0429 08:38:45.836129 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0429 08:38:45.836136 3791 shared_informer.go:356] "Caches are synced" controller="egressservices" I0429 08:38:45.836144 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0429 08:38:45.836151 3791 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0429 08:38:45.836159 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0429 08:38:45.836165 3791 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0429 08:38:45.836170 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0429 08:38:45.836174 3791 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0429 08:38:45.836178 3791 egressservice_zone.go:228] Repairing Egress Services I0429 08:38:45.836445 3791 master_controller.go:88] Starting Admin Policy Based Route Controller I0429 08:38:45.836455 3791 external_controller.go:278] Starting Admin Policy Based Route Controller I0429 08:38:45.836481 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-38.ec2.internal I0429 08:38:45.836495 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-38.ec2.internal: 20.093µs I0429 08:38:45.837162 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-176.ec2.internal I0429 08:38:45.837172 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-176.ec2.internal: 12.018µs I0429 08:38:45.837297 3791 default_network_controller.go:585] Completing all the Watchers took 306.182034ms I0429 08:38:45.837313 3791 default_network_controller.go:589] Starting unidling controllers I0429 08:38:45.837327 3791 unidle.go:46] Registering OVN SB ControllerEvent handler I0429 08:38:45.837337 3791 unidle.go:63] Populating Initial ContollerEvent events I0429 08:38:45.837350 3791 unidle.go:79] Setting up event handlers for services I0429 08:38:45.837452 3791 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0429 08:38:45.895757 3791 default_node_network_controller.go:519] Node connection status = connected I0429 08:38:46.104185 3791 default_network_controller.go:1025] Node "ip-10-0-2-176.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:38:46.395952 3791 default_node_network_controller.go:519] Node connection status = connected I0429 08:38:46.605116 3791 egressfirewall.go:723] Syncing node "ip-10-0-2-176.ec2.internal" for egress firewall I0429 08:38:46.605153 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-176.ec2.internal I0429 08:38:46.605165 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-176.ec2.internal: 17.801µs I0429 08:38:46.605181 3791 default_network_controller.go:1025] Node "ip-10-0-2-176.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:38:46.895278 3791 default_node_network_controller.go:519] Node connection status = connected I0429 08:38:47.395658 3791 default_node_network_controller.go:519] Node connection status = connected I0429 08:38:47.653238 3791 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0429 08:38:47.896267 3791 default_node_network_controller.go:519] Node connection status = connected I0429 08:38:47.914936 3791 bridgeconfig.go:537] Gateway is ready I0429 08:38:47.914955 3791 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0429 08:38:47.926522 3791 node_ip_handler_linux.go:247] Node primary address changed to 10.0.0.38. Updating OVN encap IP. I0429 08:38:47.931887 3791 node_ip_handler_linux.go:542] Will not update encap IP 10.0.0.38 - it is already configured I0429 08:38:47.931902 3791 node_ip_handler_linux.go:485] Node address changed to map[10.0.0.38/24:{} 172.20.0.1/32:{}]. Updating annotations. I0429 08:38:47.932113 3791 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.0.38/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-0-38.ec2.internal","mac-address":"12:64:c3:8c:ba:09","ip-addresses":["10.0.0.38/24"],"ip-address":"10.0.0.38/24","next-hops":["10.0.0.1"],"next-hop":"10.0.0.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:206d94dc-98da-4ade-8301-6a072e04efe5 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.0.38/24"}] on node ip-10-0-0-38.ec2.internal I0429 08:38:47.951206 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-38.ec2.internal in network "default" I0429 08:38:47.951217 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-38.ec2.internal I0429 08:38:47.951230 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-38.ec2.internal: 16.2µs I0429 08:38:47.951214 3791 egressfirewall.go:723] Syncing node "ip-10-0-0-38.ec2.internal" for egress firewall I0429 08:38:47.951264 3791 node_tracker.go:169] Node ip-10-0-0-38.ec2.internal switch + router changed, syncing services in network "default" I0429 08:38:47.951280 3791 services_controller.go:574] Full service sync requested for network=default I0429 08:38:47.951209 3791 master.go:548] Adding or Updating local node "ip-10-0-0-38.ec2.internal" for network "default" I0429 08:38:47.951736 3791 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.0.38 I0429 08:38:47.951897 3791 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-0-38.ec2.internal I0429 08:38:47.956144 3791 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-38.ec2.internal. Updating no re-route policies I0429 08:38:47.956245 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-dp8wx I0429 08:38:47.956262 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-kjqnc I0429 08:38:47.956264 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-c5n5q I0429 08:38:47.956270 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-fhh6p I0429 08:38:47.956285 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-c5n5q I0429 08:38:47.956281 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-k85l8 I0429 08:38:47.956290 3791 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-38.ec2.internal I0429 08:38:47.956299 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-k85l8 I0429 08:38:47.956303 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-mq7mq I0429 08:38:47.956308 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-9l7xh I0429 08:38:47.956307 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-996vd I0429 08:38:47.956309 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-v75l4 I0429 08:38:47.956319 3791 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-c5n5q, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-c5n5q: suppressed error logged: pod openshift-multus/network-metrics-daemon-c5n5q: no pod IPs found I0429 08:38:47.956325 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-996vd I0429 08:38:47.956327 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-k85l8 after 0 failed attempt(s) I0429 08:38:47.956327 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-v75l4 I0429 08:38:47.956344 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-9l7xh I0429 08:38:47.956263 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-h246r I0429 08:38:47.956353 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-996vd after 0 failed attempt(s) I0429 08:38:47.956297 3791 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-gph7c I0429 08:38:47.956294 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-fhh6p I0429 08:38:47.956371 3791 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-gph7c I0429 08:38:47.956310 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-dp8wx I0429 08:38:47.956403 3791 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-fhh6p] creating logical port openshift-multus_network-metrics-daemon-fhh6p for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:47.956417 3791 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-dp8wx] creating logical port openshift-network-diagnostics_network-check-target-dp8wx for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:47.956354 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-v75l4 after 0 failed attempt(s) I0429 08:38:47.956345 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-clwvz I0429 08:38:47.956463 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-clwvz I0429 08:38:47.956479 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-clwvz after 0 failed attempt(s) I0429 08:38:47.956282 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-kjqnc I0429 08:38:47.956496 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-kjqnc after 0 failed attempt(s) I0429 08:38:47.956251 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-38.ec2.internal I0429 08:38:47.956504 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-38.ec2.internal I0429 08:38:47.956514 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-38.ec2.internal after 0 failed attempt(s) I0429 08:38:47.956300 3791 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-38.ec2.internal I0429 08:38:47.956527 3791 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-0-38.ec2.internal after 0 failed attempt(s) I0429 08:38:47.956249 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-7t9xt I0429 08:38:47.956539 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-7t9xt I0429 08:38:47.956561 3791 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-7t9xt, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-7t9xt: suppressed error logged: pod openshift-network-diagnostics/network-check-target-7t9xt: no pod IPs found I0429 08:38:47.956311 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-mq7mq I0429 08:38:47.956606 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-mq7mq after 0 failed attempt(s) I0429 08:38:47.956361 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-h246r I0429 08:38:47.956625 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-h246r after 0 failed attempt(s) I0429 08:38:47.956362 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-9l7xh after 0 failed attempt(s) I0429 08:38:47.956388 3791 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-gph7c after 0 failed attempt(s) I0429 08:38:47.957325 3791 pods.go:271] [openshift-multus/network-metrics-daemon-fhh6p] addLogicalPort took 931.936µs, libovsdb time 388.532µs I0429 08:38:47.957338 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-fhh6p after 0 failed attempt(s) I0429 08:38:47.957409 3791 pods.go:271] [openshift-network-diagnostics/network-check-target-dp8wx] addLogicalPort took 1.000178ms, libovsdb time 445.621µs I0429 08:38:47.957416 3791 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-dp8wx after 0 failed attempt(s) I0429 08:38:47.973847 3791 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0429 08:38:47.984934 3791 iptables.go:108] Creating table: mangle chain: OUTPUT I0429 08:38:47.988004 3791 iptables.go:108] Creating table: nat chain: OUTPUT I0429 08:38:48.032203 3791 iptables.go:108] Creating table: nat chain: PREROUTING I0429 08:38:48.088572 3791 iptables.go:108] Creating table: filter chain: FORWARD I0429 08:38:48.105681 3791 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0429 08:38:48.105776 3791 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0429 08:38:48.105823 3791 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0429 08:38:48.105905 3791 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0429 08:38:48.105951 3791 gateway_init.go:325] Gateway and management port readiness took 2.714950525s I0429 08:38:48.105979 3791 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0429 08:38:48.106030 3791 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0429 08:38:48.106042 3791 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0429 08:38:48.106050 3791 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0429 08:38:48.106103 3791 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0429 08:38:48.106197 3791 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 65.998µs I0429 08:38:48.106387 3791 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0429 08:38:48.106412 3791 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 15.826µs I0429 08:38:48.106421 3791 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 32.797µs I0429 08:38:48.106429 3791 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0429 08:38:48.106440 3791 nad_controller.go:379] [node-nad-controller NAD controller]: started I0429 08:38:48.106445 3791 default_node_network_controller.go:899] Starting the default node network controller I0429 08:38:48.106452 3791 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0429 08:38:48.106533 3791 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.226µs I0429 08:38:48.106580 3791 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0429 08:38:48.106589 3791 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0429 08:38:48.106610 3791 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000131e80 0xc0001c3ce0 0xc0001c3dc0 0xc0001c3e30 0xc0001c3f10 0xc0002ec000 0xc0002ec070 0xc0004f5540 0xc0002ec0e0 0xc0002ec150 0xc000c000f0 0xc0064fef50 0xc0002ec1c0 map[0x2ff9080:0xc00370f560 0x2ff95c0:0xc004d85da0 0x2ffa2e0:0xc00333e120 0x2ffa580:0xc00370f500 0x2ffad60:0xc00370f5c0 0x2ffb000:0xc00370f620 0x2ffb2a0:0xc004d85d40 0x2ffba80:0xc004d85e00 0x2ffbd20:0xc004d85e60 0x3026980:0xc001681680 0x3026d20:0xc0016816e0 0x3027800:0xc000aab740 0x3029160:0xc001a9c000 0x30298a0:0xc00333e0c0 0x302a720:0xc000baa000] 0xc00016aa10 0} I0429 08:38:48.106678 3791 gateway.go:151] Starting gateway service sync I0429 08:38:48.107072 3791 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0429 08:38:48.109211 3791 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0429 08:38:48.136497 3791 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0429 08:38:48.137569 3791 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0429 08:38:48.138653 3791 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0429 08:38:48.170288 3791 gateway.go:168] Gateway service sync done. Time taken: 63.597648ms I0429 08:38:48.170604 3791 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30172 I0429 08:38:48.171088 3791 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000131e80 0xc0001c3ce0 0xc0001c3dc0 0xc0001c3e30 0xc0001c3f10 0xc0002ec000 0xc0002ec070 0xc0004f5540 0xc0002ec0e0 0xc0002ec150 0xc000c000f0 0xc0064fef50 0xc0002ec1c0 map[0x2ff9080:0xc00370f560 0x2ff95c0:0xc004d85da0 0x2ffa2e0:0xc00333e120 0x2ffa580:0xc00370f500 0x2ffad60:0xc00370f5c0 0x2ffb000:0xc00370f620 0x2ffb2a0:0xc004d85d40 0x2ffba80:0xc004d85e00 0x2ffbd20:0xc004d85e60 0x3026980:0xc001681680 0x3026d20:0xc0016816e0 0x3027800:0xc000aab740 0x3029160:0xc001a9c000 0x30298a0:0xc00333e0c0 0x302a720:0xc000baa000] 0xc00016aa10 0} I0429 08:38:48.171561 3791 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc000131e80 0xc0001c3ce0 0xc0001c3dc0 0xc0001c3e30 0xc0001c3f10 0xc0002ec000 0xc0002ec070 0xc0004f5540 0xc0002ec0e0 0xc0002ec150 0xc000c000f0 0xc0064fef50 0xc0002ec1c0 map[0x2ff9080:0xc00370f560 0x2ff95c0:0xc004d85da0 0x2ffa2e0:0xc00333e120 0x2ffa580:0xc00370f500 0x2ffad60:0xc00370f5c0 0x2ffb000:0xc00370f620 0x2ffb2a0:0xc004d85d40 0x2ffba80:0xc004d85e00 0x2ffbd20:0xc004d85e60 0x3026980:0xc001681680 0x3026d20:0xc0016816e0 0x3027800:0xc000aab740 0x3029160:0xc001a9c000 0x30298a0:0xc00333e0c0 0x302a720:0xc000baa000] 0xc00016aa10 0} I0429 08:38:48.171609 3791 gateway.go:532] Reconciling gateway with updates I0429 08:38:48.177141 3791 gateway.go:339] Spawning Conntrack Rule Check Thread I0429 08:38:48.177194 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 08:38:48.177209 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 08:38:48.177212 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 08:38:48.177227 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 08:38:48.177232 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 08:38:48.177243 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 08:38:48.177250 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 08:38:48.177249 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 08:38:48.177265 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 08:38:48.177193 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0429 08:38:48.177278 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0429 08:38:48.177274 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 08:38:48.177278 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0429 08:38:48.177291 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 08:38:48.177291 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 08:38:48.177299 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 08:38:48.177303 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0429 08:38:48.177315 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 08:38:48.177316 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0429 08:38:48.177315 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 08:38:48.177326 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0429 08:38:48.177328 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 08:38:48.177333 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 08:38:48.177334 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 08:38:48.177336 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 08:38:48.177333 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 08:38:48.177342 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 08:38:48.177347 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 08:38:48.177350 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 08:38:48.177353 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0429 08:38:48.177354 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 08:38:48.177363 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0429 08:38:48.177369 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 08:38:48.177216 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0429 08:38:48.177267 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 08:38:48.177398 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0429 08:38:48.177334 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 08:38:48.177403 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0429 08:38:48.177322 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 08:38:48.177435 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0429 08:38:48.177342 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0429 08:38:48.177376 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0429 08:38:48.177413 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0429 08:38:48.177442 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 08:38:48.177442 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0429 08:38:48.177461 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 08:38:48.177355 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0429 08:38:48.177471 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0429 08:38:48.177483 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0429 08:38:48.177453 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0429 08:38:48.177503 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0429 08:38:48.177654 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 08:38:48.177665 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 08:38:48.177671 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 08:38:48.177673 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0429 08:38:48.177675 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0429 08:38:48.177679 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 08:38:48.177682 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0429 08:38:48.177686 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0429 08:38:48.177691 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0429 08:38:48.177704 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0429 08:38:48.177704 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 08:38:48.177704 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0429 08:38:48.177664 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0429 08:38:48.177713 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0429 08:38:48.177716 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 08:38:48.177719 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0429 08:38:48.177716 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 08:38:48.177721 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0429 08:38:48.177729 3791 port_claim.go:71] Svc openshift-ingress/router-default: port 32258 is already open, no action needed I0429 08:38:48.177728 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 08:38:48.177731 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0429 08:38:48.177721 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0429 08:38:48.177736 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 08:38:48.177741 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0429 08:38:48.177740 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0429 08:38:48.177742 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 08:38:48.177721 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 08:38:48.177747 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0429 08:38:48.177751 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0429 08:38:48.177752 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 08:38:48.177753 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 08:38:48.177757 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0429 08:38:48.177760 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 08:38:48.177759 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 08:38:48.177759 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 08:38:48.177765 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0429 08:38:48.177769 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 08:38:48.177772 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 08:38:48.177775 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0429 08:38:48.177776 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0429 08:38:48.177779 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0429 08:38:48.177780 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 08:38:48.177759 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0429 08:38:48.177789 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 08:38:48.177728 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 08:38:48.177808 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 08:38:48.177813 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0429 08:38:48.177735 3791 port_claim.go:71] Svc openshift-ingress/router-default: port 32323 is already open, no action needed I0429 08:38:48.177732 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0429 08:38:48.177708 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0429 08:38:48.177721 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 08:38:48.177738 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 08:38:48.177880 3791 default_node_network_controller.go:944] Gateway and management port readiness took 71.282387ms I0429 08:38:48.177746 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 08:38:48.177748 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0429 08:38:48.177748 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 08:38:48.177920 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 08:38:48.177751 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 08:38:48.177714 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0429 08:38:48.177971 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0429 08:38:48.177759 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 08:38:48.177981 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 08:38:48.177731 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 08:38:48.177879 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 08:38:48.177742 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 08:38:48.178040 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 08:38:48.178734 3791 node_ip_handler_linux.go:160] Node IP manager is running W0429 08:38:48.214336 3791 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0429 08:38:48.214516 3791 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.128.0.1 on link ovn-k8s-mp0 with MTU: 0 W0429 08:38:48.214667 3791 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:80:00:01/10.128.0.1) on link ovn-k8s-mp0 I0429 08:38:48.223306 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0429 08:38:48.242707 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0429 08:38:48.242717 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0429 08:38:48.242732 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0429 08:38:48.242739 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0429 08:38:48.242745 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0429 08:38:48.242750 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0429 08:38:48.283435 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0429 08:38:48.283446 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0429 08:38:48.283463 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0429 08:38:48.283471 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0429 08:38:48.283477 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0429 08:38:48.283482 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0429 08:38:48.283488 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0429 08:38:48.283493 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0429 08:38:48.283522 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0429 08:38:48.283545 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0429 08:38:48.283553 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) W0429 08:38:50.207377 3791 egressip_healthcheck.go:76] Health checking using insecure connection I0429 08:38:50.207421 3791 default_node_network_controller.go:1380] Starting node controller node sync I0429 08:38:50.207425 3791 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.128.0.2:9107 I0429 08:38:50.218700 3791 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.264196ms I0429 08:38:50.218919 3791 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0429 08:38:50.219231 3791 egressservice_node.go:93] Setting up event handlers for Egress Services I0429 08:38:50.219345 3791 egressservice_node.go:183] Starting Egress Services Controller I0429 08:38:50.219360 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0429 08:38:50.219370 3791 shared_informer.go:356] "Caches are synced" controller="egressservices" I0429 08:38:50.219376 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0429 08:38:50.219381 3791 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0429 08:38:50.219387 3791 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0429 08:38:50.219391 3791 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0429 08:38:50.219395 3791 egressservice_node.go:197] Repairing Egress Services I0429 08:38:50.229211 3791 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0429 08:38:50.229224 3791 external_controller.go:278] Starting Admin Policy Based Route Controller I0429 08:38:50.229329 3791 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0429 08:38:50.229339 3791 link_network_manager.go:119] Link manager is running I0429 08:38:50.229346 3791 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0429 08:38:50.229504 3791 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0429 08:38:50.230075 3791 vrf_manager.go:128] VRF manager is running I0429 08:38:50.230190 3791 node_controller_manager.go:437] Removing flows to drop GARP I0429 08:38:50.230204 3791 gateway.go:532] Reconciling gateway with updates I0429 08:38:50.244822 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0429 08:38:50.244855 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 08:38:50.244866 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0429 08:38:50.244869 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 08:38:50.244880 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0429 08:38:50.244884 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 08:38:50.244859 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0429 08:38:50.244890 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 08:38:50.244893 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 08:38:50.244908 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 08:38:50.244910 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 08:38:50.244911 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 08:38:50.244921 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0429 08:38:50.244922 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 08:38:50.244869 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0429 08:38:50.244925 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 08:38:50.244947 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 08:38:50.244948 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 08:38:50.244948 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 08:38:50.244960 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 08:38:50.244889 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 08:38:50.244971 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 08:38:50.244976 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0429 08:38:50.244842 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0429 08:38:50.244983 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0429 08:38:50.244896 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0429 08:38:50.244998 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0429 08:38:50.245003 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0429 08:38:50.245006 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0429 08:38:50.244984 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 08:38:50.245017 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 08:38:50.245020 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0429 08:38:50.245022 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 08:38:50.245022 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 08:38:50.245028 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0429 08:38:50.245007 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 08:38:50.245035 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0429 08:38:50.245042 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 08:38:50.245045 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 08:38:50.245052 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 08:38:50.245059 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 08:38:50.245062 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 08:38:50.245060 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 08:38:50.245067 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0429 08:38:50.245070 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 08:38:50.245073 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 08:38:50.245076 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0429 08:38:50.244896 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 08:38:50.245083 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 08:38:50.245089 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 08:38:50.245093 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 08:38:50.245027 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 08:38:50.245104 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 08:38:50.244896 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 08:38:50.245083 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 08:38:50.245113 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 08:38:50.245120 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 08:38:50.245124 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 08:38:50.245125 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0429 08:38:50.244934 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0429 08:38:50.245127 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0429 08:38:50.245005 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 08:38:50.245047 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0429 08:38:50.245141 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0429 08:38:50.245144 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 08:38:50.245147 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 08:38:50.245146 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 08:38:50.245152 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 08:38:50.245157 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 08:38:50.244875 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 08:38:50.245159 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0429 08:38:50.245113 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 08:38:50.245171 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 08:38:50.245175 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 08:38:50.245177 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 08:38:50.245182 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 08:38:50.245186 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 08:38:50.245186 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0429 08:38:50.245189 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0429 08:38:50.245159 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 08:38:50.245196 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0429 08:38:50.245205 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 08:38:50.245208 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 08:38:50.245210 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0429 08:38:50.245217 3791 port_claim.go:71] Svc openshift-ingress/router-default: port 32258 is already open, no action needed I0429 08:38:50.245211 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0429 08:38:50.245222 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 08:38:50.245226 3791 port_claim.go:71] Svc openshift-ingress/router-default: port 32323 is already open, no action needed I0429 08:38:50.245012 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0429 08:38:50.245188 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 08:38:50.245037 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 08:38:50.245281 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 08:38:50.245172 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 08:38:50.245308 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 08:38:50.244994 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0429 08:38:50.245394 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 08:38:50.245407 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 08:38:50.245394 3791 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0429 08:38:50.245465 3791 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0429 08:38:50.252387 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0429 08:38:50.252396 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0429 08:38:50.252408 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0429 08:38:50.252416 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0429 08:38:50.252423 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0429 08:38:50.252430 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0429 08:38:50.252435 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0429 08:38:50.252440 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0429 08:38:50.252445 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0429 08:38:50.252449 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0429 08:38:50.252474 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0429 08:38:50.252482 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0429 08:38:50.252487 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0429 08:38:50.293575 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0429 08:38:50.293594 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0429 08:38:50.293602 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0429 08:38:50.293610 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0429 08:38:50.293618 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0429 08:38:50.293627 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0429 08:38:50.293635 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0429 08:38:50.293643 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0429 08:38:50.293651 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0429 08:38:50.293656 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0429 08:38:50.293662 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0429 08:38:50.293669 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0429 08:38:50.333847 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0429 08:38:50.333861 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0429 08:38:50.333851 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0429 08:38:50.333874 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0429 08:38:50.333883 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0429 08:38:50.333895 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0429 08:38:50.333904 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0429 08:38:50.333920 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0429 08:38:50.333929 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0429 08:38:50.333935 3791 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0429 08:38:52.638687 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-38.ec2.internal I0429 08:38:52.638718 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-38.ec2.internal: 36.24µs I0429 08:38:52.715149 3791 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-c64d887fc-nvgqf] creating logical port openshift-image-registry_image-registry-c64d887fc-nvgqf for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.715282 3791 kube.go:256] Updating pod openshift-image-registry/image-registry-c64d887fc-nvgqf I0429 08:38:52.736764 3791 pod.go:62] [openshift-image-registry/image-registry-c64d887fc-nvgqf] pod update took 21.51633ms I0429 08:38:52.736789 3791 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-c64d887fc-nvgqf] addLogicalPort annotation time took 21.545089ms I0429 08:38:52.738157 3791 pods.go:271] [openshift-image-registry/image-registry-c64d887fc-nvgqf] addLogicalPort took 23.025854ms, libovsdb time 1.085354ms I0429 08:38:52.752120 3791 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-hxggd] creating logical port openshift-dns_dns-default-hxggd for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.752219 3791 kube.go:256] Updating pod openshift-dns/dns-default-hxggd I0429 08:38:52.771115 3791 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-4dfp4] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-4dfp4 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.771193 3791 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-4dfp4 I0429 08:38:52.785255 3791 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-798868845-mtmkt] creating logical port openshift-ingress_router-default-798868845-mtmkt for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.785357 3791 kube.go:256] Updating pod openshift-ingress/router-default-798868845-mtmkt I0429 08:38:52.785432 3791 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-m9h7x] creating logical port openshift-network-diagnostics_network-check-source-84fd6d9d54-m9h7x for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.785518 3791 kube.go:256] Updating pod openshift-network-diagnostics/network-check-source-84fd6d9d54-m9h7x I0429 08:38:52.788051 3791 pod.go:62] [openshift-dns/dns-default-hxggd] pod update took 35.859648ms I0429 08:38:52.788071 3791 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-hxggd] addLogicalPort annotation time took 35.881246ms I0429 08:38:52.789245 3791 pods.go:271] [openshift-dns/dns-default-hxggd] addLogicalPort took 37.142578ms, libovsdb time 906.078µs I0429 08:38:52.789577 3791 base_network_controller_pods.go:487] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-2ld57] creating logical port openshift-deployment-validation-operator_deployment-validation-operator-catalog-2ld57 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.789667 3791 kube.go:256] Updating pod openshift-deployment-validation-operator/deployment-validation-operator-catalog-2ld57 I0429 08:38:52.789692 3791 base_network_controller_pods.go:487] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-jhkr6] creating logical port openshift-service-ca-operator_service-ca-operator-bc9564db4-jhkr6 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.789819 3791 kube.go:256] Updating pod openshift-service-ca-operator/service-ca-operator-bc9564db4-jhkr6 I0429 08:38:52.809362 3791 base_network_controller_pods.go:487] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-d5z8v] creating logical port openshift-cluster-samples-operator_cluster-samples-operator-67f4d568c7-d5z8v for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.809448 3791 kube.go:256] Updating pod openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-d5z8v I0429 08:38:52.814141 3791 base_network_controller_pods.go:487] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-xxpw6] creating logical port openshift-cluster-storage-operator_volume-data-source-validator-6944588dcc-xxpw6 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.814248 3791 kube.go:256] Updating pod openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-xxpw6 I0429 08:38:52.814419 3791 base_network_controller_pods.go:487] [default/openshift-console-operator/console-operator-66dbbb6468-7brmw] creating logical port openshift-console-operator_console-operator-66dbbb6468-7brmw for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.814483 3791 base_network_controller_pods.go:487] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-q474x] creating logical port openshift-network-console_networking-console-plugin-7f449dfbfd-q474x for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.814517 3791 kube.go:256] Updating pod openshift-console-operator/console-operator-66dbbb6468-7brmw I0429 08:38:52.814546 3791 kube.go:256] Updating pod openshift-network-console/networking-console-plugin-7f449dfbfd-q474x I0429 08:38:52.814556 3791 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-675b7c69bc-txw8m] creating logical port openshift-ingress_router-default-675b7c69bc-txw8m for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.814636 3791 kube.go:256] Updating pod openshift-ingress/router-default-675b7c69bc-txw8m I0429 08:38:52.843143 3791 pod.go:62] [openshift-network-diagnostics/network-check-source-84fd6d9d54-m9h7x] pod update took 57.647151ms I0429 08:38:52.843161 3791 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-source-84fd6d9d54-m9h7x] addLogicalPort annotation time took 57.66647ms I0429 08:38:52.843240 3791 pod.go:62] [openshift-service-ca-operator/service-ca-operator-bc9564db4-jhkr6] pod update took 53.473009ms I0429 08:38:52.843256 3791 base_network_controller_pods.go:951] [default/openshift-service-ca-operator/service-ca-operator-bc9564db4-jhkr6] addLogicalPort annotation time took 53.490787ms I0429 08:38:52.846921 3791 pods.go:271] [openshift-service-ca-operator/service-ca-operator-bc9564db4-jhkr6] addLogicalPort took 57.237983ms, libovsdb time 3.453696ms I0429 08:38:52.847070 3791 base_network_controller_pods.go:487] [default/openshift-insights/insights-operator-6dbb575d6c-w8kdg] creating logical port openshift-insights_insights-operator-6dbb575d6c-w8kdg for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.847198 3791 kube.go:256] Updating pod openshift-insights/insights-operator-6dbb575d6c-w8kdg I0429 08:38:52.847482 3791 pods.go:271] [openshift-network-diagnostics/network-check-source-84fd6d9d54-m9h7x] addLogicalPort took 62.058311ms, libovsdb time 4.103701ms I0429 08:38:52.858146 3791 base_network_controller_pods.go:487] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-4vkpf] creating logical port openshift-monitoring_cluster-monitoring-operator-557b74df6-4vkpf for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:52.858273 3791 kube.go:256] Updating pod openshift-monitoring/cluster-monitoring-operator-557b74df6-4vkpf I0429 08:38:52.977738 3791 pod.go:62] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2ld57] pod update took 188.087589ms I0429 08:38:52.977882 3791 base_network_controller_pods.go:951] [default/openshift-deployment-validation-operator/deployment-validation-operator-catalog-2ld57] addLogicalPort annotation time took 188.234422ms I0429 08:38:52.977740 3791 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-4dfp4] pod update took 206.56273ms I0429 08:38:52.977960 3791 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-4dfp4] addLogicalPort annotation time took 206.784678ms I0429 08:38:52.979348 3791 pods.go:271] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2ld57] addLogicalPort took 189.779059ms, libovsdb time 1.149361ms I0429 08:38:52.979574 3791 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-4dfp4] addLogicalPort took 208.464525ms, libovsdb time 1.369939ms I0429 08:38:53.020671 3791 pod.go:62] [openshift-console-operator/console-operator-66dbbb6468-7brmw] pod update took 206.169443ms I0429 08:38:53.020690 3791 base_network_controller_pods.go:951] [default/openshift-console-operator/console-operator-66dbbb6468-7brmw] addLogicalPort annotation time took 206.191453ms I0429 08:38:53.021629 3791 pods.go:271] [openshift-console-operator/console-operator-66dbbb6468-7brmw] addLogicalPort took 207.229277ms, libovsdb time 695.917µs I0429 08:38:53.021963 3791 pod.go:62] [openshift-network-console/networking-console-plugin-7f449dfbfd-q474x] pod update took 207.43729ms I0429 08:38:53.021972 3791 pod.go:62] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-d5z8v] pod update took 212.549452ms I0429 08:38:53.021979 3791 base_network_controller_pods.go:951] [default/openshift-network-console/networking-console-plugin-7f449dfbfd-q474x] addLogicalPort annotation time took 207.453906ms I0429 08:38:53.021990 3791 base_network_controller_pods.go:951] [default/openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-d5z8v] addLogicalPort annotation time took 212.566178ms I0429 08:38:53.021974 3791 pod.go:62] [openshift-insights/insights-operator-6dbb575d6c-w8kdg] pod update took 174.815287ms I0429 08:38:53.022054 3791 pod.go:62] [openshift-ingress/router-default-798868845-mtmkt] pod update took 236.725199ms I0429 08:38:53.022066 3791 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-798868845-mtmkt] addLogicalPort annotation time took 236.739534ms I0429 08:38:53.022056 3791 base_network_controller_pods.go:951] [default/openshift-insights/insights-operator-6dbb575d6c-w8kdg] addLogicalPort annotation time took 174.900541ms I0429 08:38:53.022754 3791 pods.go:271] [openshift-network-console/networking-console-plugin-7f449dfbfd-q474x] addLogicalPort took 208.281591ms, libovsdb time 558.182µs I0429 08:38:53.023165 3791 pods.go:271] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-d5z8v] addLogicalPort took 213.809455ms, libovsdb time 951.211µs I0429 08:38:53.023611 3791 pods.go:271] [openshift-ingress/router-default-798868845-mtmkt] addLogicalPort took 238.369962ms, libovsdb time 1.345948ms I0429 08:38:53.023644 3791 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-lrtr5] creating logical port openshift-ingress-canary_ingress-canary-lrtr5 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:53.023738 3791 pods.go:271] [openshift-insights/insights-operator-6dbb575d6c-w8kdg] addLogicalPort took 176.694063ms, libovsdb time 1.415271ms I0429 08:38:53.023798 3791 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-lrtr5 I0429 08:38:53.065952 3791 pod.go:62] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-xxpw6] pod update took 251.739749ms I0429 08:38:53.065975 3791 base_network_controller_pods.go:951] [default/openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-xxpw6] addLogicalPort annotation time took 251.765322ms I0429 08:38:53.065952 3791 pod.go:62] [openshift-monitoring/cluster-monitoring-operator-557b74df6-4vkpf] pod update took 207.71059ms I0429 08:38:53.066059 3791 base_network_controller_pods.go:951] [default/openshift-monitoring/cluster-monitoring-operator-557b74df6-4vkpf] addLogicalPort annotation time took 207.825739ms I0429 08:38:53.067272 3791 pods.go:271] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-xxpw6] addLogicalPort took 253.137606ms, libovsdb time 963.452µs I0429 08:38:53.067490 3791 pods.go:271] [openshift-monitoring/cluster-monitoring-operator-557b74df6-4vkpf] addLogicalPort took 209.364756ms, libovsdb time 1.150052ms I0429 08:38:53.068406 3791 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-7sdzj] creating logical port openshift-kube-storage-version-migrator-operator_kube-storage-version-migrator-operator-74478b59c6-7sdzj for pod on switch ip-10-0-0-38.ec2.internal I0429 08:38:53.068467 3791 kube.go:256] Updating pod openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-7sdzj I0429 08:38:53.111262 3791 pod.go:62] [openshift-ingress/router-default-675b7c69bc-txw8m] pod update took 296.646172ms I0429 08:38:53.111282 3791 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-675b7c69bc-txw8m] addLogicalPort annotation time took 296.668934ms I0429 08:38:53.112292 3791 pods.go:271] [openshift-ingress/router-default-675b7c69bc-txw8m] addLogicalPort took 297.746637ms, libovsdb time 737.152µs I0429 08:38:53.168003 3791 pod.go:62] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-7sdzj] pod update took 99.553947ms I0429 08:38:53.168019 3791 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-7sdzj] addLogicalPort annotation time took 99.57362ms I0429 08:38:53.168106 3791 pod.go:62] [openshift-ingress-canary/ingress-canary-lrtr5] pod update took 144.331984ms I0429 08:38:53.168123 3791 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-lrtr5] addLogicalPort annotation time took 144.351846ms I0429 08:38:53.168914 3791 pods.go:271] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-7sdzj] addLogicalPort took 100.516364ms, libovsdb time 640.547µs I0429 08:38:53.169197 3791 pods.go:271] [openshift-ingress-canary/ingress-canary-lrtr5] addLogicalPort took 145.558197ms, libovsdb time 828.012µs I0429 08:38:53.606056 3791 cni.go:452] [openshift-ingress-canary/ingress-canary-lrtr5 7f8969d5c49d466201dffa379eaeadd565fdf792ccd9b0bffe6040e84b76dcad network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-lrtr5 7f8969d5c49d466201dffa379eaeadd565fdf792ccd9b0bffe6040e84b76dcad network default NAD default NAD key ] I0429 08:38:53.606845 3791 cni.go:452] [openshift-network-diagnostics/network-check-source-84fd6d9d54-m9h7x c5997cf9fef02fffd6b0fba8f5a37709503fca287612e13ec7e48cc25a7c3e88 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-m9h7x c5997cf9fef02fffd6b0fba8f5a37709503fca287612e13ec7e48cc25a7c3e88 network default NAD default NAD key ] I0429 08:38:53.607043 3791 cni.go:452] [openshift-service-ca-operator/service-ca-operator-bc9564db4-jhkr6 98bc7ab59faaf330cd1f203058816e0d4b617a8032797737247de74a5655acad network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-jhkr6 98bc7ab59faaf330cd1f203058816e0d4b617a8032797737247de74a5655acad network default NAD default NAD key ] I0429 08:38:53.626345 3791 cni.go:452] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2ld57 2978123813c42d0063922bf44fd67a4e279c9afeab3300db3845594513652d33 network default NAD default NAD key ] ADD starting CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2ld57 2978123813c42d0063922bf44fd67a4e279c9afeab3300db3845594513652d33 network default NAD default NAD key ] I0429 08:38:53.648906 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-source-84fd6d9d54-m9h7x, hostIfaceName: c5997cf9fef02ff, network: default, NAD default, SandboxID: "c5997cf9fef02fffd6b0fba8f5a37709503fca287612e13ec7e48cc25a7c3e88", PCI device ID: , UID: "de7b8301-bdcf-4940-ab06-879916caf23c", MAC: 0a:58:0a:80:00:09, IPs: [10.128.0.9/23] I0429 08:38:53.648963 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-lrtr5, hostIfaceName: 7f8969d5c49d466, network: default, NAD default, SandboxID: "7f8969d5c49d466201dffa379eaeadd565fdf792ccd9b0bffe6040e84b76dcad", PCI device ID: , UID: "f794e0c8-303f-438d-bc1b-78b96b84d0ca", MAC: 0a:58:0a:80:00:13, IPs: [10.128.0.19/23] I0429 08:38:53.649204 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-deployment-validation-operator, podName: deployment-validation-operator-catalog-2ld57, hostIfaceName: 2978123813c42d0, network: default, NAD default, SandboxID: "2978123813c42d0063922bf44fd67a4e279c9afeab3300db3845594513652d33", PCI device ID: , UID: "d55350c5-f2c3-43d8-bfbc-79ff2eb82632", MAC: 0a:58:0a:80:00:0a, IPs: [10.128.0.10/23] I0429 08:38:53.655272 3791 cni.go:452] [openshift-insights/insights-operator-6dbb575d6c-w8kdg 422754ef11cd282cd3553538bdaed86ca685fa928757ac90e4859f38f6644e48 network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-operator-6dbb575d6c-w8kdg 422754ef11cd282cd3553538bdaed86ca685fa928757ac90e4859f38f6644e48 network default NAD default NAD key ] I0429 08:38:53.661684 3791 cni.go:452] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-xxpw6 312abcadcb79cfff04eb284ed8b2669df6a3ee491093cd8118fc60a0fde83664 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-xxpw6 312abcadcb79cfff04eb284ed8b2669df6a3ee491093cd8118fc60a0fde83664 network default NAD default NAD key ] I0429 08:38:53.664621 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca-operator, podName: service-ca-operator-bc9564db4-jhkr6, hostIfaceName: 98bc7ab59faaf33, network: default, NAD default, SandboxID: "98bc7ab59faaf330cd1f203058816e0d4b617a8032797737247de74a5655acad", PCI device ID: , UID: "ae86421a-0653-44b1-84ed-bd5e88df3b80", MAC: 0a:58:0a:80:00:0b, IPs: [10.128.0.11/23] I0429 08:38:53.666433 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-operator-6dbb575d6c-w8kdg, hostIfaceName: 422754ef11cd282, network: default, NAD default, SandboxID: "422754ef11cd282cd3553538bdaed86ca685fa928757ac90e4859f38f6644e48", PCI device ID: , UID: "f87193f8-8f49-4369-904d-4ca58fa39bd0", MAC: 0a:58:0a:80:00:11, IPs: [10.128.0.17/23] I0429 08:38:53.681039 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-storage-operator, podName: volume-data-source-validator-6944588dcc-xxpw6, hostIfaceName: 312abcadcb79cff, network: default, NAD default, SandboxID: "312abcadcb79cfff04eb284ed8b2669df6a3ee491093cd8118fc60a0fde83664", PCI device ID: , UID: "87c577be-e0bf-47c8-9041-93674714578d", MAC: 0a:58:0a:80:00:0d, IPs: [10.128.0.13/23] I0429 08:38:53.692155 3791 cni.go:452] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-7sdzj 12051f1dfea5d426566784f419c0f277393a24adb06f491db57749ed8cc36863 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-7sdzj 12051f1dfea5d426566784f419c0f277393a24adb06f491db57749ed8cc36863 network default NAD default NAD key ] I0429 08:38:53.698670 3791 cni.go:452] [openshift-console-operator/console-operator-66dbbb6468-7brmw 3f9571c33d7e710dd8a208667e9dd8720c1bfd72d860a1db2de6e6645da175a8 network default NAD default NAD key ] ADD starting CNI request [openshift-console-operator/console-operator-66dbbb6468-7brmw 3f9571c33d7e710dd8a208667e9dd8720c1bfd72d860a1db2de6e6645da175a8 network default NAD default NAD key ] I0429 08:38:53.716120 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator-operator, podName: kube-storage-version-migrator-operator-74478b59c6-7sdzj, hostIfaceName: 12051f1dfea5d42, network: default, NAD default, SandboxID: "12051f1dfea5d426566784f419c0f277393a24adb06f491db57749ed8cc36863", PCI device ID: , UID: "ae84d818-614d-48fa-b0cd-a3a0f704179a", MAC: 0a:58:0a:80:00:14, IPs: [10.128.0.20/23] I0429 08:38:53.719307 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-console-operator, podName: console-operator-66dbbb6468-7brmw, hostIfaceName: 3f9571c33d7e710, network: default, NAD default, SandboxID: "3f9571c33d7e710dd8a208667e9dd8720c1bfd72d860a1db2de6e6645da175a8", PCI device ID: , UID: "b9882d82-f845-4675-a9ff-6b0c047b88a7", MAC: 0a:58:0a:80:00:0e, IPs: [10.128.0.14/23] I0429 08:38:53.728635 3791 cni.go:473] [openshift-network-diagnostics/network-check-source-84fd6d9d54-m9h7x c5997cf9fef02fffd6b0fba8f5a37709503fca287612e13ec7e48cc25a7c3e88 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-source-84fd6d9d54-m9h7x c5997cf9fef02fffd6b0fba8f5a37709503fca287612e13ec7e48cc25a7c3e88 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"96:70:da:fc:d4:46\",\"name\":\"c5997cf9fef02ff\"},{\"mac\":\"0a:58:0a:80:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/af36dc0e-0d26-47a7-a57e-3ffdc01e2a21\"}],\"ips\":[{\"address\":\"10.128.0.9/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:38:53.739586 3791 cni.go:473] [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2ld57 2978123813c42d0063922bf44fd67a4e279c9afeab3300db3845594513652d33 network default NAD default NAD key default] ADD finished CNI request [openshift-deployment-validation-operator/deployment-validation-operator-catalog-2ld57 2978123813c42d0063922bf44fd67a4e279c9afeab3300db3845594513652d33 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"32:96:22:4c:5f:61\",\"name\":\"2978123813c42d0\"},{\"mac\":\"0a:58:0a:80:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1ee5a366-ceaf-4132-b6da-784f4f3e9f61\"}],\"ips\":[{\"address\":\"10.128.0.10/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:38:53.742578 3791 cni.go:473] [openshift-insights/insights-operator-6dbb575d6c-w8kdg 422754ef11cd282cd3553538bdaed86ca685fa928757ac90e4859f38f6644e48 network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-operator-6dbb575d6c-w8kdg 422754ef11cd282cd3553538bdaed86ca685fa928757ac90e4859f38f6644e48 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:e8:80:88:20:61\",\"name\":\"422754ef11cd282\"},{\"mac\":\"0a:58:0a:80:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/422bbf6e-195d-4593-898f-55252546f0e1\"}],\"ips\":[{\"address\":\"10.128.0.17/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:38:53.744432 3791 cni.go:473] [openshift-service-ca-operator/service-ca-operator-bc9564db4-jhkr6 98bc7ab59faaf330cd1f203058816e0d4b617a8032797737247de74a5655acad network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca-operator/service-ca-operator-bc9564db4-jhkr6 98bc7ab59faaf330cd1f203058816e0d4b617a8032797737247de74a5655acad network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:c0:1a:15:e3:40\",\"name\":\"98bc7ab59faaf33\"},{\"mac\":\"0a:58:0a:80:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/68aedb93-9430-4664-978c-249b729c376f\"}],\"ips\":[{\"address\":\"10.128.0.11/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:38:53.745895 3791 cni.go:473] [openshift-ingress-canary/ingress-canary-lrtr5 7f8969d5c49d466201dffa379eaeadd565fdf792ccd9b0bffe6040e84b76dcad network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-lrtr5 7f8969d5c49d466201dffa379eaeadd565fdf792ccd9b0bffe6040e84b76dcad network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"2e:a3:93:d1:1a:a0\",\"name\":\"7f8969d5c49d466\"},{\"mac\":\"0a:58:0a:80:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0b08991f-aecd-480b-a157-61e505b9151f\"}],\"ips\":[{\"address\":\"10.128.0.19/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:38:53.758129 3791 cni.go:473] [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-xxpw6 312abcadcb79cfff04eb284ed8b2669df6a3ee491093cd8118fc60a0fde83664 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-storage-operator/volume-data-source-validator-6944588dcc-xxpw6 312abcadcb79cfff04eb284ed8b2669df6a3ee491093cd8118fc60a0fde83664 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:0a:9a:49:1a:7b\",\"name\":\"312abcadcb79cff\"},{\"mac\":\"0a:58:0a:80:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/85ed8af6-8c06-4152-a3cc-6fa15b364902\"}],\"ips\":[{\"address\":\"10.128.0.13/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:38:53.772597 3791 cni.go:473] [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-7sdzj 12051f1dfea5d426566784f419c0f277393a24adb06f491db57749ed8cc36863 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator-operator/kube-storage-version-migrator-operator-74478b59c6-7sdzj 12051f1dfea5d426566784f419c0f277393a24adb06f491db57749ed8cc36863 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"56:f4:4b:5b:fa:07\",\"name\":\"12051f1dfea5d42\"},{\"mac\":\"0a:58:0a:80:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/903e0e13-2f6f-4cc1-a9b5-5b1db5d408b0\"}],\"ips\":[{\"address\":\"10.128.0.20/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:38:53.778442 3791 cni.go:473] [openshift-console-operator/console-operator-66dbbb6468-7brmw 3f9571c33d7e710dd8a208667e9dd8720c1bfd72d860a1db2de6e6645da175a8 network default NAD default NAD key default] ADD finished CNI request [openshift-console-operator/console-operator-66dbbb6468-7brmw 3f9571c33d7e710dd8a208667e9dd8720c1bfd72d860a1db2de6e6645da175a8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:eb:41:91:35:80\",\"name\":\"3f9571c33d7e710\"},{\"mac\":\"0a:58:0a:80:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a6d7356c-f017-4e44-a722-d36130ca41d3\"}],\"ips\":[{\"address\":\"10.128.0.14/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:38:55.035612 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-240.ec2.internal in network "default" I0429 08:38:55.035634 3791 node_tracker.go:220] Node ip-10-0-1-240.ec2.internal has invalid / no HostSubnet annotations (probably waiting on initialization), or it's a hybrid overlay node: could not find "k8s.ovn.org/node-subnets" annotation I0429 08:38:55.035663 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-240.ec2.internal E0429 08:38:55.035696 3791 default_network_controller.go:833] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-240.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation I0429 08:38:55.035681 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-240.ec2.internal: 20.309µs I0429 08:38:55.035661 3791 egressfirewall.go:723] Syncing node "ip-10-0-1-240.ec2.internal" for egress firewall I0429 08:38:55.035714 3791 obj_retry.go:578] Failed to create *v1.Node ip-10-0-1-240.ec2.internal, error: [adding or updating remote node chassis ip-10-0-1-240.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-1-240.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-1-240.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-1-240.ec2.internal, error parsing annotation for node ip-10-0-1-240.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation] I0429 08:38:55.047587 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false E0429 08:38:55.047595 3791 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-240.ec2.internal annotation: unexpected end of JSON input E0429 08:38:55.047629 3791 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-240.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0429 08:38:55.058637 3791 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-240.ec2.internal annotation: unexpected end of JSON input I0429 08:38:55.058724 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: true E0429 08:38:55.058764 3791 default_network_controller.go:1042] Failed to add node IPs to openshift-host-network address_set: error parsing annotation for node ip-10-0-1-240.ec2.internal: suppressed error logged: could not find "k8s.ovn.org/node-subnets" annotation E0429 08:38:55.085624 3791 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-240.ec2.internal annotation: unexpected end of JSON input E0429 08:38:55.085671 3791 subnet_annotations.go:147] Failed to unmarshal old node ip-10-0-1-240.ec2.internal annotation: unexpected end of JSON input I0429 08:38:55.085693 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:38:55.102571 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:38:55.102587 3791 egressfirewall.go:723] Syncing node "ip-10-0-1-240.ec2.internal" for egress firewall I0429 08:38:55.102591 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-240.ec2.internal I0429 08:38:55.102612 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-240.ec2.internal: 26.63µs I0429 08:38:55.307768 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-240.ec2.internal I0429 08:38:55.307787 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-240.ec2.internal: 22.874µs I0429 08:38:55.307767 3791 egressfirewall.go:723] Syncing node "ip-10-0-1-240.ec2.internal" for egress firewall I0429 08:38:55.307796 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:38:55.331109 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:38:55.556263 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:38:58.313579 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:39:01.825162 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-176.ec2.internal in network "default" I0429 08:39:01.825221 3791 node_tracker.go:235] Node ip-10-0-2-176.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-2-176.ec2.internal" I0429 08:39:01.825218 3791 default_network_controller.go:1025] Node "ip-10-0-2-176.ec2.internal" in remote zone "ip-10-0-2-176.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:39:01.825237 3791 node_tracker.go:169] Node ip-10-0-2-176.ec2.internal switch + router changed, syncing services in network "default" I0429 08:39:01.825252 3791 services_controller.go:574] Full service sync requested for network=default I0429 08:39:01.981098 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-176.ec2.internal in network "default" I0429 08:39:01.981160 3791 node_tracker.go:169] Node ip-10-0-2-176.ec2.internal switch + router changed, syncing services in network "default" I0429 08:39:01.981179 3791 services_controller.go:574] Full service sync requested for network=default I0429 08:39:02.015993 3791 default_network_controller.go:1025] Node "ip-10-0-2-176.ec2.internal" in remote zone "ip-10-0-2-176.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:39:02.017019 3791 zone_ic_handler.go:195] Time taken to create transit switch: 328.437µs I0429 08:39:02.017041 3791 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-2-176.ec2.internal for the network default I0429 08:39:02.018200 3791 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-2-176.ec2.internal" on network "default" took: 1.538397ms W0429 08:39:02.031326 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0429 08:39:02.132239 3791 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 08:39:02.791805 3791 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0429 08:39:02.792858 3791 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 1.030594ms I0429 08:39:02.810138 3791 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0429 08:39:02.898758 3791 base_network_controller_pods.go:487] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-skz8z] creating logical port openshift-kube-storage-version-migrator_migrator-846d9b8bdc-skz8z for pod on switch ip-10-0-0-38.ec2.internal I0429 08:39:02.898867 3791 kube.go:256] Updating pod openshift-kube-storage-version-migrator/migrator-846d9b8bdc-skz8z I0429 08:39:02.920798 3791 pod.go:62] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-skz8z] pod update took 21.967794ms I0429 08:39:02.920841 3791 base_network_controller_pods.go:951] [default/openshift-kube-storage-version-migrator/migrator-846d9b8bdc-skz8z] addLogicalPort annotation time took 22.00487ms I0429 08:39:02.922276 3791 pods.go:271] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-skz8z] addLogicalPort took 23.53242ms, libovsdb time 999.153µs I0429 08:39:03.199021 3791 namespace.go:98] [openshift-service-ca] adding namespace I0429 08:39:03.199952 3791 namespace.go:102] [openshift-service-ca] adding namespace took 902.868µs I0429 08:39:03.216145 3791 namespace.go:142] [openshift-service-ca] updating namespace I0429 08:39:03.256110 3791 cni.go:452] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-skz8z 0292af166d686581bb668f8f6048c375608fc0f66b7825031a2892c856c5b402 network default NAD default NAD key ] ADD starting CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-skz8z 0292af166d686581bb668f8f6048c375608fc0f66b7825031a2892c856c5b402 network default NAD default NAD key ] I0429 08:39:03.276314 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-kube-storage-version-migrator, podName: migrator-846d9b8bdc-skz8z, hostIfaceName: 0292af166d68658, network: default, NAD default, SandboxID: "0292af166d686581bb668f8f6048c375608fc0f66b7825031a2892c856c5b402", PCI device ID: , UID: "940612d0-d4d4-48bd-a116-051c2240c906", MAC: 0a:58:0a:80:00:15, IPs: [10.128.0.21/23] I0429 08:39:03.325170 3791 cni.go:473] [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-skz8z 0292af166d686581bb668f8f6048c375608fc0f66b7825031a2892c856c5b402 network default NAD default NAD key default] ADD finished CNI request [openshift-kube-storage-version-migrator/migrator-846d9b8bdc-skz8z 0292af166d686581bb668f8f6048c375608fc0f66b7825031a2892c856c5b402 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:e4:dd:31:cb:43\",\"name\":\"0292af166d68658\"},{\"mac\":\"0a:58:0a:80:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/db1a1995-def7-4d03-a550-ad3f518558b5\"}],\"ips\":[{\"address\":\"10.128.0.21/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:04.540213 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-176.ec2.internal in network "default" I0429 08:39:04.540243 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-176.ec2.internal I0429 08:39:04.540269 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-176.ec2.internal: 24.688µs I0429 08:39:04.540296 3791 egressfirewall.go:723] Syncing node "ip-10-0-2-176.ec2.internal" for egress firewall I0429 08:39:04.540310 3791 node_tracker.go:169] Node ip-10-0-2-176.ec2.internal switch + router changed, syncing services in network "default" I0429 08:39:04.540325 3791 services_controller.go:574] Full service sync requested for network=default I0429 08:39:04.540362 3791 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-176.ec2.internal", IP: 10.0.2.176 to PMTUD blocking rules I0429 08:39:04.540379 3791 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-176.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0429 08:39:04.540619 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0429 08:39:04.540721 3791 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 08:39:04.546408 3791 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-176.ec2.internal. Updating no re-route policies I0429 08:39:04.555683 3791 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-176.ec2.internal", IP: 10.0.2.176 to PMTUD blocking rules I0429 08:39:04.555698 3791 default_node_network_controller.go:1293] Adding remote node "ip-10-0-2-176.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0429 08:39:04.883182 3791 base_network_controller_pods.go:487] [default/openshift-service-ca/service-ca-d9cb48497-ft9kb] creating logical port openshift-service-ca_service-ca-d9cb48497-ft9kb for pod on switch ip-10-0-0-38.ec2.internal I0429 08:39:04.883314 3791 kube.go:256] Updating pod openshift-service-ca/service-ca-d9cb48497-ft9kb I0429 08:39:04.900011 3791 pod.go:62] [openshift-service-ca/service-ca-d9cb48497-ft9kb] pod update took 16.720774ms I0429 08:39:04.900037 3791 base_network_controller_pods.go:951] [default/openshift-service-ca/service-ca-d9cb48497-ft9kb] addLogicalPort annotation time took 16.750243ms I0429 08:39:04.901278 3791 pods.go:271] [openshift-service-ca/service-ca-d9cb48497-ft9kb] addLogicalPort took 18.111873ms, libovsdb time 887.107µs I0429 08:39:05.202340 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:39:05.240594 3791 cni.go:452] [openshift-service-ca/service-ca-d9cb48497-ft9kb bbfe4ae3d4b35300e4fd796162f98d402677c1fe2ccabb9ed61fa9f42306bd54 network default NAD default NAD key ] ADD starting CNI request [openshift-service-ca/service-ca-d9cb48497-ft9kb bbfe4ae3d4b35300e4fd796162f98d402677c1fe2ccabb9ed61fa9f42306bd54 network default NAD default NAD key ] I0429 08:39:05.250284 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-service-ca, podName: service-ca-d9cb48497-ft9kb, hostIfaceName: bbfe4ae3d4b3530, network: default, NAD default, SandboxID: "bbfe4ae3d4b35300e4fd796162f98d402677c1fe2ccabb9ed61fa9f42306bd54", PCI device ID: , UID: "73ea07f5-f4fd-4b0b-a8a4-e65c458d21f9", MAC: 0a:58:0a:80:00:16, IPs: [10.128.0.22/23] I0429 08:39:05.295523 3791 cni.go:473] [openshift-service-ca/service-ca-d9cb48497-ft9kb bbfe4ae3d4b35300e4fd796162f98d402677c1fe2ccabb9ed61fa9f42306bd54 network default NAD default NAD key default] ADD finished CNI request [openshift-service-ca/service-ca-d9cb48497-ft9kb bbfe4ae3d4b35300e4fd796162f98d402677c1fe2ccabb9ed61fa9f42306bd54 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:6f:28:54:7a:de\",\"name\":\"bbfe4ae3d4b3530\"},{\"mac\":\"0a:58:0a:80:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/cf767d06-4bbc-4eb3-812f-7fb9bb89efe6\"}],\"ips\":[{\"address\":\"10.128.0.22/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:05.964995 3791 egressfirewall.go:723] Syncing node "ip-10-0-1-240.ec2.internal" for egress firewall I0429 08:39:05.965009 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-240.ec2.internal I0429 08:39:05.965010 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:39:05.965025 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-240.ec2.internal: 21.024µs I0429 08:39:07.971570 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-176.ec2.internal I0429 08:39:07.971604 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-176.ec2.internal: 40.662µs I0429 08:39:08.923284 3791 cni.go:452] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-d5z8v 3f24a37976054bad15d693b884e15fc68299bf3ea855115272ed12b0ac7eaa58 network default NAD default NAD key ] ADD starting CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-d5z8v 3f24a37976054bad15d693b884e15fc68299bf3ea855115272ed12b0ac7eaa58 network default NAD default NAD key ] I0429 08:39:08.932492 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-cluster-samples-operator, podName: cluster-samples-operator-67f4d568c7-d5z8v, hostIfaceName: 3f24a37976054ba, network: default, NAD default, SandboxID: "3f24a37976054bad15d693b884e15fc68299bf3ea855115272ed12b0ac7eaa58", PCI device ID: , UID: "9e4c09ea-4beb-4b1d-98d1-606e21bd0c72", MAC: 0a:58:0a:80:00:0c, IPs: [10.128.0.12/23] I0429 08:39:08.977870 3791 cni.go:473] [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-d5z8v 3f24a37976054bad15d693b884e15fc68299bf3ea855115272ed12b0ac7eaa58 network default NAD default NAD key default] ADD finished CNI request [openshift-cluster-samples-operator/cluster-samples-operator-67f4d568c7-d5z8v 3f24a37976054bad15d693b884e15fc68299bf3ea855115272ed12b0ac7eaa58 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:90:44:77:43:10\",\"name\":\"3f24a37976054ba\"},{\"mac\":\"0a:58:0a:80:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bd7618b4-d521-4322-ac58-af83180fde9e\"}],\"ips\":[{\"address\":\"10.128.0.12/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:39:10.052543 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0429 08:39:10.476675 3791 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-84cdb56c78-7fcwk IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-84cdb56c78-7fcwk: no pod IPs found I0429 08:39:12.015323 3791 obj_retry.go:498] Detected object openshift-image-registry/image-registry-c64d887fc-nvgqf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:39:12.015354 3791 pods.go:176] Deleting pod: openshift-image-registry/image-registry-c64d887fc-nvgqf I0429 08:39:12.016772 3791 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-c64d887fc-nvgqf, ips: 10.128.0.5 I0429 08:39:12.016799 3791 obj_retry.go:498] Detected object openshift-image-registry/image-registry-c64d887fc-nvgqf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:39:12.027479 3791 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-6d695d8795-7w94b] creating logical port openshift-image-registry_image-registry-6d695d8795-7w94b for pod on switch ip-10-0-0-38.ec2.internal I0429 08:39:12.027566 3791 kube.go:256] Updating pod openshift-image-registry/image-registry-6d695d8795-7w94b I0429 08:39:12.035329 3791 obj_retry.go:492] Detected object openshift-image-registry/image-registry-c64d887fc-nvgqf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:39:12.035350 3791 obj_retry.go:492] Detected object openshift-image-registry/image-registry-c64d887fc-nvgqf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:39:12.047337 3791 pod.go:62] [openshift-image-registry/image-registry-6d695d8795-7w94b] pod update took 19.797711ms I0429 08:39:12.047361 3791 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-6d695d8795-7w94b] addLogicalPort annotation time took 19.826533ms I0429 08:39:12.048458 3791 pods.go:271] [openshift-image-registry/image-registry-6d695d8795-7w94b] addLogicalPort took 20.990043ms, libovsdb time 806.134µs I0429 08:39:12.051919 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-c64d887fc-nvgqf I0429 08:39:12.051942 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-c64d887fc-nvgqf I0429 08:39:12.416701 3791 cni.go:452] [openshift-image-registry/image-registry-6d695d8795-7w94b 48a5a39d50f9bbd1869f1836bb954437f523fa5d5b26a2cdb040ff61ee3aa338 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-6d695d8795-7w94b 48a5a39d50f9bbd1869f1836bb954437f523fa5d5b26a2cdb040ff61ee3aa338 network default NAD default NAD key ] I0429 08:39:12.425907 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-6d695d8795-7w94b, hostIfaceName: 48a5a39d50f9bbd, network: default, NAD default, SandboxID: "48a5a39d50f9bbd1869f1836bb954437f523fa5d5b26a2cdb040ff61ee3aa338", PCI device ID: , UID: "84d3796f-88e3-47a2-9454-d5761bdc730f", MAC: 0a:58:0a:80:00:17, IPs: [10.128.0.23/23] I0429 08:39:12.472218 3791 cni.go:473] [openshift-image-registry/image-registry-6d695d8795-7w94b 48a5a39d50f9bbd1869f1836bb954437f523fa5d5b26a2cdb040ff61ee3aa338 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-6d695d8795-7w94b 48a5a39d50f9bbd1869f1836bb954437f523fa5d5b26a2cdb040ff61ee3aa338 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:35:cc:87:aa:2b\",\"name\":\"48a5a39d50f9bbd\"},{\"mac\":\"0a:58:0a:80:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/733eb0b3-8e32-4956-8c3f-2ccfa7160804\"}],\"ips\":[{\"address\":\"10.128.0.23/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:15.448496 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:39:17.957521 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-n692f I0429 08:39:17.957547 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-n692f I0429 08:39:17.957575 3791 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-m7r85 I0429 08:39:17.957583 3791 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-n692f, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-n692f: suppressed error logged: pod openshift-multus/network-metrics-daemon-n692f: no pod IPs found I0429 08:39:17.957594 3791 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-m7r85 I0429 08:39:17.957626 3791 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-m7r85, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-m7r85: suppressed error logged: pod openshift-network-diagnostics/network-check-target-m7r85: no pod IPs found I0429 08:39:21.725723 3791 cni.go:452] [openshift-multus/network-metrics-daemon-fhh6p 9d9c1c32cfd46390febb085f8a46e70f9f5991ae489ea6ff6ee49f0769221ab0 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-fhh6p 9d9c1c32cfd46390febb085f8a46e70f9f5991ae489ea6ff6ee49f0769221ab0 network default NAD default NAD key ] I0429 08:39:21.735286 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-fhh6p, hostIfaceName: 9d9c1c32cfd4639, network: default, NAD default, SandboxID: "9d9c1c32cfd46390febb085f8a46e70f9f5991ae489ea6ff6ee49f0769221ab0", PCI device ID: , UID: "f6cdb8b0-415a-40e0-851d-ea7829e34dde", MAC: 0a:58:0a:80:00:04, IPs: [10.128.0.4/23] I0429 08:39:21.783198 3791 cni.go:473] [openshift-multus/network-metrics-daemon-fhh6p 9d9c1c32cfd46390febb085f8a46e70f9f5991ae489ea6ff6ee49f0769221ab0 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-fhh6p 9d9c1c32cfd46390febb085f8a46e70f9f5991ae489ea6ff6ee49f0769221ab0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:a0:cf:a2:94:1d\",\"name\":\"9d9c1c32cfd4639\"},{\"mac\":\"0a:58:0a:80:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/35d11062-abe4-4401-b6f3-e01d06041d71\"}],\"ips\":[{\"address\":\"10.128.0.4/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:22.017803 3791 cni.go:452] [openshift-network-diagnostics/network-check-target-dp8wx 49c827c5d3cbe855d21fd70fcba7f4ea4d5f8eeb4ba996bd0bc86ad959383fd1 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-dp8wx 49c827c5d3cbe855d21fd70fcba7f4ea4d5f8eeb4ba996bd0bc86ad959383fd1 network default NAD default NAD key ] I0429 08:39:22.027424 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-dp8wx, hostIfaceName: 49c827c5d3cbe85, network: default, NAD default, SandboxID: "49c827c5d3cbe855d21fd70fcba7f4ea4d5f8eeb4ba996bd0bc86ad959383fd1", PCI device ID: , UID: "7fa2b821-80b1-48c3-9788-ea758c410b98", MAC: 0a:58:0a:80:00:03, IPs: [10.128.0.3/23] I0429 08:39:22.072436 3791 cni.go:473] [openshift-network-diagnostics/network-check-target-dp8wx 49c827c5d3cbe855d21fd70fcba7f4ea4d5f8eeb4ba996bd0bc86ad959383fd1 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-dp8wx 49c827c5d3cbe855d21fd70fcba7f4ea4d5f8eeb4ba996bd0bc86ad959383fd1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"62:c1:aa:e7:4e:ec\",\"name\":\"49c827c5d3cbe85\"},{\"mac\":\"0a:58:0a:80:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e05ebc91-12cc-400e-9e38-81281844ad77\"}],\"ips\":[{\"address\":\"10.128.0.3/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:24.975538 3791 cni.go:452] [openshift-dns/dns-default-hxggd d53111f8358f19f9d5ccf82651d3514c7d10cea0b861b42cc98a4bb8f2854380 network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-hxggd d53111f8358f19f9d5ccf82651d3514c7d10cea0b861b42cc98a4bb8f2854380 network default NAD default NAD key ] I0429 08:39:24.984593 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-hxggd, hostIfaceName: d53111f8358f19f, network: default, NAD default, SandboxID: "d53111f8358f19f9d5ccf82651d3514c7d10cea0b861b42cc98a4bb8f2854380", PCI device ID: , UID: "49c2ee62-71fa-4b26-aae7-f06e27341a45", MAC: 0a:58:0a:80:00:06, IPs: [10.128.0.6/23] I0429 08:39:25.007924 3791 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-4dfp4 6364765afc77eb897aef4c9fa92f1f37204ebe231d02c477d6f618bdd52ca84c network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-4dfp4 6364765afc77eb897aef4c9fa92f1f37204ebe231d02c477d6f618bdd52ca84c network default NAD default NAD key ] I0429 08:39:25.017029 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-4dfp4, hostIfaceName: 6364765afc77eb8, network: default, NAD default, SandboxID: "6364765afc77eb897aef4c9fa92f1f37204ebe231d02c477d6f618bdd52ca84c", PCI device ID: , UID: "fde5fd8c-89eb-4f90-8391-b611ffcb7306", MAC: 0a:58:0a:80:00:07, IPs: [10.128.0.7/23] I0429 08:39:25.033941 3791 cni.go:473] [openshift-dns/dns-default-hxggd d53111f8358f19f9d5ccf82651d3514c7d10cea0b861b42cc98a4bb8f2854380 network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-hxggd d53111f8358f19f9d5ccf82651d3514c7d10cea0b861b42cc98a4bb8f2854380 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:85:4d:26:d5:f6\",\"name\":\"d53111f8358f19f\"},{\"mac\":\"0a:58:0a:80:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/bb001db0-9108-4f7c-acbe-6dfdf64916c7\"}],\"ips\":[{\"address\":\"10.128.0.6/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:25.046284 3791 cni.go:452] [openshift-ingress/router-default-798868845-mtmkt d7505b41af8f25ac8f927e5cddd1cbdb37494cdc98aaac1d2b1bd0f7525cc5e4 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-798868845-mtmkt d7505b41af8f25ac8f927e5cddd1cbdb37494cdc98aaac1d2b1bd0f7525cc5e4 network default NAD default NAD key ] I0429 08:39:25.062898 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-798868845-mtmkt, hostIfaceName: d7505b41af8f25a, network: default, NAD default, SandboxID: "d7505b41af8f25ac8f927e5cddd1cbdb37494cdc98aaac1d2b1bd0f7525cc5e4", PCI device ID: , UID: "72a6bca7-1e28-408e-9396-b799dd2a0a78", MAC: 0a:58:0a:80:00:08, IPs: [10.128.0.8/23] I0429 08:39:25.067445 3791 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-4dfp4 6364765afc77eb897aef4c9fa92f1f37204ebe231d02c477d6f618bdd52ca84c network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-4dfp4 6364765afc77eb897aef4c9fa92f1f37204ebe231d02c477d6f618bdd52ca84c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:0a:2e:d2:d2:60\",\"name\":\"6364765afc77eb8\"},{\"mac\":\"0a:58:0a:80:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5702edfe-83ce-4e05-8359-5b39482b16b0\"}],\"ips\":[{\"address\":\"10.128.0.7/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:25.109717 3791 cni.go:473] [openshift-ingress/router-default-798868845-mtmkt d7505b41af8f25ac8f927e5cddd1cbdb37494cdc98aaac1d2b1bd0f7525cc5e4 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-798868845-mtmkt d7505b41af8f25ac8f927e5cddd1cbdb37494cdc98aaac1d2b1bd0f7525cc5e4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:ef:e1:39:96:8d\",\"name\":\"d7505b41af8f25a\"},{\"mac\":\"0a:58:0a:80:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6b928a4c-2ca1-45c6-9374-d7d5d54b2446\"}],\"ips\":[{\"address\":\"10.128.0.8/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:25.206478 3791 cni.go:452] [openshift-network-console/networking-console-plugin-7f449dfbfd-q474x f9843fd551c6fec28c5fc3c64480c7ef53518f80a232188136f89c9b0e3ca443 network default NAD default NAD key ] ADD starting CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-q474x f9843fd551c6fec28c5fc3c64480c7ef53518f80a232188136f89c9b0e3ca443 network default NAD default NAD key ] I0429 08:39:25.231596 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-console, podName: networking-console-plugin-7f449dfbfd-q474x, hostIfaceName: f9843fd551c6fec, network: default, NAD default, SandboxID: "f9843fd551c6fec28c5fc3c64480c7ef53518f80a232188136f89c9b0e3ca443", PCI device ID: , UID: "f27ae014-5352-46c8-81de-4881bac26943", MAC: 0a:58:0a:80:00:0f, IPs: [10.128.0.15/23] I0429 08:39:25.262059 3791 cni.go:452] [openshift-ingress/router-default-675b7c69bc-txw8m 079bff5339c8c53e03730b543b6dc4925bed5290c348425a7e294b509816b022 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-675b7c69bc-txw8m 079bff5339c8c53e03730b543b6dc4925bed5290c348425a7e294b509816b022 network default NAD default NAD key ] I0429 08:39:25.277504 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-675b7c69bc-txw8m, hostIfaceName: 079bff5339c8c53, network: default, NAD default, SandboxID: "079bff5339c8c53e03730b543b6dc4925bed5290c348425a7e294b509816b022", PCI device ID: , UID: "d89bd55b-958d-4ec1-821f-c24e78f2ec0d", MAC: 0a:58:0a:80:00:10, IPs: [10.128.0.16/23] I0429 08:39:25.283641 3791 cni.go:473] [openshift-network-console/networking-console-plugin-7f449dfbfd-q474x f9843fd551c6fec28c5fc3c64480c7ef53518f80a232188136f89c9b0e3ca443 network default NAD default NAD key default] ADD finished CNI request [openshift-network-console/networking-console-plugin-7f449dfbfd-q474x f9843fd551c6fec28c5fc3c64480c7ef53518f80a232188136f89c9b0e3ca443 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6e:b0:f5:ab:8a:49\",\"name\":\"f9843fd551c6fec\"},{\"mac\":\"0a:58:0a:80:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/008cd66e-6a34-43f8-8b16-39e8323809c6\"}],\"ips\":[{\"address\":\"10.128.0.15/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:25.297299 3791 cni.go:452] [openshift-monitoring/cluster-monitoring-operator-557b74df6-4vkpf a1ae403f1748777969ff331d4b5d51093848e7d79dbdbd8f732f1136f236d531 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-4vkpf a1ae403f1748777969ff331d4b5d51093848e7d79dbdbd8f732f1136f236d531 network default NAD default NAD key ] I0429 08:39:25.308630 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: cluster-monitoring-operator-557b74df6-4vkpf, hostIfaceName: a1ae403f1748777, network: default, NAD default, SandboxID: "a1ae403f1748777969ff331d4b5d51093848e7d79dbdbd8f732f1136f236d531", PCI device ID: , UID: "27b2511e-507c-438d-a9d3-4755aa12e3bb", MAC: 0a:58:0a:80:00:12, IPs: [10.128.0.18/23] I0429 08:39:25.331995 3791 cni.go:473] [openshift-ingress/router-default-675b7c69bc-txw8m 079bff5339c8c53e03730b543b6dc4925bed5290c348425a7e294b509816b022 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-675b7c69bc-txw8m 079bff5339c8c53e03730b543b6dc4925bed5290c348425a7e294b509816b022 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b6:68:68:d0:3a:54\",\"name\":\"079bff5339c8c53\"},{\"mac\":\"0a:58:0a:80:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ba7252ef-59df-4a55-b63d-ec94dc29bc37\"}],\"ips\":[{\"address\":\"10.128.0.16/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:25.356700 3791 cni.go:473] [openshift-monitoring/cluster-monitoring-operator-557b74df6-4vkpf a1ae403f1748777969ff331d4b5d51093848e7d79dbdbd8f732f1136f236d531 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/cluster-monitoring-operator-557b74df6-4vkpf a1ae403f1748777969ff331d4b5d51093848e7d79dbdbd8f732f1136f236d531 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:a5:59:3d:1a:e1\",\"name\":\"a1ae403f1748777\"},{\"mac\":\"0a:58:0a:80:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/71a23431-ab03-4af4-8a18-60f0869fc8b6\"}],\"ips\":[{\"address\":\"10.128.0.18/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:25.792385 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0429 08:39:29.636669 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 50051: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 08:39:29.652523 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0429 08:39:29.652995 3791 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 08:39:29.653013 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 08:39:29.653025 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0429 08:39:29.653030 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 08:39:29.712809 3791 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0429 08:39:29.768701 3791 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0429 08:39:29.795548 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0429 08:39:29.830341 3791 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0429 08:39:29.879989 3791 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-image-registry/image-registry-6d695d8795-dvkbm IP address from the namespace address-set, err: pod openshift-image-registry/image-registry-6d695d8795-dvkbm: no pod IPs found I0429 08:39:29.895773 3791 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-snbcd] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-snbcd for pod on switch ip-10-0-0-38.ec2.internal I0429 08:39:29.895907 3791 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-snbcd I0429 08:39:29.914749 3791 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-snbcd] pod update took 18.874588ms I0429 08:39:29.914768 3791 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-snbcd] addLogicalPort annotation time took 18.898846ms I0429 08:39:29.915865 3791 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-snbcd] addLogicalPort took 20.104635ms, libovsdb time 771.129µs I0429 08:39:29.920693 3791 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-46fv4] creating logical port openshift-insights_insights-runtime-extractor-46fv4 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:39:29.920790 3791 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-46fv4 I0429 08:39:29.940863 3791 pod.go:62] [openshift-insights/insights-runtime-extractor-46fv4] pod update took 20.099137ms I0429 08:39:29.940881 3791 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-46fv4] addLogicalPort annotation time took 20.120503ms I0429 08:39:29.941954 3791 pods.go:271] [openshift-insights/insights-runtime-extractor-46fv4] addLogicalPort took 21.270499ms, libovsdb time 704.968µs I0429 08:39:30.255473 3791 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-snbcd db749c2124dbf5be37c13bab216df7501fb029c07f59181ff701e49b2c6cb253 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-snbcd db749c2124dbf5be37c13bab216df7501fb029c07f59181ff701e49b2c6cb253 network default NAD default NAD key ] I0429 08:39:30.264613 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-snbcd, hostIfaceName: db749c2124dbf5b, network: default, NAD default, SandboxID: "db749c2124dbf5be37c13bab216df7501fb029c07f59181ff701e49b2c6cb253", PCI device ID: , UID: "f1357d75-ec94-42d9-be65-94ceb853e8f1", MAC: 0a:58:0a:80:00:18, IPs: [10.128.0.24/23] I0429 08:39:30.284214 3791 cni.go:452] [openshift-insights/insights-runtime-extractor-46fv4 d30720aeed5f5a338e487904d6059b3cf7b1e2a80b5fa03ab2fbbb91c2ab6a1d network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-46fv4 d30720aeed5f5a338e487904d6059b3cf7b1e2a80b5fa03ab2fbbb91c2ab6a1d network default NAD default NAD key ] I0429 08:39:30.295018 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-46fv4, hostIfaceName: d30720aeed5f5a3, network: default, NAD default, SandboxID: "d30720aeed5f5a338e487904d6059b3cf7b1e2a80b5fa03ab2fbbb91c2ab6a1d", PCI device ID: , UID: "de175e62-0a87-4556-9fca-1d3b9c35621d", MAC: 0a:58:0a:80:00:19, IPs: [10.128.0.25/23] I0429 08:39:30.317002 3791 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-snbcd db749c2124dbf5be37c13bab216df7501fb029c07f59181ff701e49b2c6cb253 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-snbcd db749c2124dbf5be37c13bab216df7501fb029c07f59181ff701e49b2c6cb253 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:ed:76:d9:e1:02\",\"name\":\"db749c2124dbf5b\"},{\"mac\":\"0a:58:0a:80:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/af403581-cb4a-46d0-9f0d-ec5a6c99df3a\"}],\"ips\":[{\"address\":\"10.128.0.24/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:30.342911 3791 cni.go:473] [openshift-insights/insights-runtime-extractor-46fv4 d30720aeed5f5a338e487904d6059b3cf7b1e2a80b5fa03ab2fbbb91c2ab6a1d network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-46fv4 d30720aeed5f5a338e487904d6059b3cf7b1e2a80b5fa03ab2fbbb91c2ab6a1d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:b9:a2:15:67:6b\",\"name\":\"d30720aeed5f5a3\"},{\"mac\":\"0a:58:0a:80:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4ba3f2b7-cd92-4771-93fe-0b0155fb99e8\"}],\"ips\":[{\"address\":\"10.128.0.25/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:39:31.102139 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-240.ec2.internal in network "default" I0429 08:39:31.102222 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "ip-10-0-1-240.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:39:31.102244 3791 node_tracker.go:235] Node ip-10-0-1-240.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-1-240.ec2.internal" I0429 08:39:31.102267 3791 node_tracker.go:169] Node ip-10-0-1-240.ec2.internal switch + router changed, syncing services in network "default" I0429 08:39:31.102284 3791 services_controller.go:574] Full service sync requested for network=default W0429 08:39:31.102531 3791 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 08:39:31.102590 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0429 08:39:31.102772 3791 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0429 08:39:31.281475 3791 default_network_controller.go:1025] Node "ip-10-0-1-240.ec2.internal" in remote zone "ip-10-0-1-240.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 08:39:31.281503 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-240.ec2.internal in network "default" I0429 08:39:31.281555 3791 node_tracker.go:169] Node ip-10-0-1-240.ec2.internal switch + router changed, syncing services in network "default" I0429 08:39:31.281567 3791 services_controller.go:574] Full service sync requested for network=default W0429 08:39:31.281884 3791 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 08:39:31.281939 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0429 08:39:31.282029 3791 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0429 08:39:31.282508 3791 zone_ic_handler.go:195] Time taken to create transit switch: 283.341µs I0429 08:39:31.282520 3791 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-240.ec2.internal for the network default I0429 08:39:31.283762 3791 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-240.ec2.internal" on network "default" took: 1.549659ms I0429 08:39:31.846854 3791 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-240.ec2.internal in network "default" I0429 08:39:31.846913 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-240.ec2.internal I0429 08:39:31.846925 3791 egressfirewall.go:723] Syncing node "ip-10-0-1-240.ec2.internal" for egress firewall I0429 08:39:31.846929 3791 node_tracker.go:169] Node ip-10-0-1-240.ec2.internal switch + router changed, syncing services in network "default" I0429 08:39:31.846958 3791 services_controller.go:574] Full service sync requested for network=default I0429 08:39:31.846962 3791 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-240.ec2.internal", IP: 10.0.1.240 to PMTUD blocking rules I0429 08:39:31.846973 3791 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-240.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0429 08:39:31.846928 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-240.ec2.internal: 19.544µs W0429 08:39:31.847223 3791 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 08:39:31.847289 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics W0429 08:39:31.847433 3791 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0429 08:39:31.853103 3791 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-240.ec2.internal. Updating no re-route policies I0429 08:39:31.869663 3791 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-240.ec2.internal", IP: 10.0.1.240 to PMTUD blocking rules I0429 08:39:31.869679 3791 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-240.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules W0429 08:39:32.816629 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0429 08:39:33.119053 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0429 08:39:34.014469 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0429 08:39:36.269750 3791 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-240.ec2.internal I0429 08:39:36.269781 3791 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-240.ec2.internal: 39.67µs W0429 08:39:37.073643 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 08:39:37.073661 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0429 08:39:37.073667 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 08:39:42.892656 3791 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 08:39:42.892671 3791 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 08:39:42.892678 3791 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 08:39:43.177369 3791 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 08:39:43.177386 3791 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 08:39:43.177391 3791 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0429 08:39:45.784641 3791 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:39:45.784746 3791 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0429 08:39:45.848618 3791 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 63.911467ms I0429 08:39:45.848705 3791 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 63.998653ms I0429 08:39:45.851628 3791 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 66.998571ms, libovsdb time 2.46896ms I0429 08:39:46.151673 3791 cni.go:452] [openshift-monitoring/prometheus-k8s-1 d60cf2fe3eb950e4c960515176840df86cd4a82e839c8599b689c238761b2090 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 d60cf2fe3eb950e4c960515176840df86cd4a82e839c8599b689c238761b2090 network default NAD default NAD key ] I0429 08:39:46.161372 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: d60cf2fe3eb950e, network: default, NAD default, SandboxID: "d60cf2fe3eb950e4c960515176840df86cd4a82e839c8599b689c238761b2090", PCI device ID: , UID: "f9554d6c-326f-4671-843f-53a779f1505e", MAC: 0a:58:0a:80:00:1a, IPs: [10.128.0.26/23] I0429 08:39:46.211499 3791 cni.go:473] [openshift-monitoring/prometheus-k8s-1 d60cf2fe3eb950e4c960515176840df86cd4a82e839c8599b689c238761b2090 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 d60cf2fe3eb950e4c960515176840df86cd4a82e839c8599b689c238761b2090 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:4b:df:47:3a:85\",\"name\":\"d60cf2fe3eb950e\"},{\"mac\":\"0a:58:0a:80:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8cbd423e-02ed-441a-90cf-b02d3efdd45b\"}],\"ips\":[{\"address\":\"10.128.0.26/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:39:46.905108 3791 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0429 08:39:47.170716 3791 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0429 08:39:47.234091 3791 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:39:47.234110 3791 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:39:47.926046 3791 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 08:39:47.926065 3791 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 08:39:47.926070 3791 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 08:39:47.926074 3791 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 08:39:47.953421 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0429 08:39:48.190764 3791 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:39:48.190782 3791 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:39:49.247106 3791 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 08:39:49.247125 3791 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 08:39:49.247130 3791 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 08:39:49.247134 3791 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 08:39:49.291045 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0429 08:39:49.303333 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-console-operator/metrics W0429 08:39:54.051904 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target I0429 08:39:55.110466 3791 obj_retry.go:498] Detected object openshift-image-registry/image-registry-c64d887fc-tbtcv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:39:55.111030 3791 obj_retry.go:498] Detected object openshift-image-registry/image-registry-c64d887fc-tbtcv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:39:55.974215 3791 obj_retry.go:492] Detected object openshift-image-registry/image-registry-c64d887fc-tbtcv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:39:55.974237 3791 obj_retry.go:492] Detected object openshift-image-registry/image-registry-c64d887fc-tbtcv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:39:55.978270 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-c64d887fc-tbtcv I0429 08:39:55.978290 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-c64d887fc-tbtcv W0429 08:40:04.010030 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0429 08:40:07.806821 3791 base_network_controller_pods.go:487] [default/openshift-console/console-67f858855d-4nzvz] creating logical port openshift-console_console-67f858855d-4nzvz for pod on switch ip-10-0-0-38.ec2.internal I0429 08:40:07.806972 3791 kube.go:256] Updating pod openshift-console/console-67f858855d-4nzvz I0429 08:40:07.827134 3791 pod.go:62] [openshift-console/console-67f858855d-4nzvz] pod update took 20.194198ms I0429 08:40:07.827155 3791 base_network_controller_pods.go:951] [default/openshift-console/console-67f858855d-4nzvz] addLogicalPort annotation time took 20.218667ms I0429 08:40:07.828402 3791 pods.go:271] [openshift-console/console-67f858855d-4nzvz] addLogicalPort took 21.59591ms, libovsdb time 879.351µs I0429 08:40:08.163695 3791 cni.go:452] [openshift-console/console-67f858855d-4nzvz a764270d8e23fad72578f31e519ca569fb215564f8493a714040778e74ec1061 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-67f858855d-4nzvz a764270d8e23fad72578f31e519ca569fb215564f8493a714040778e74ec1061 network default NAD default NAD key ] I0429 08:40:08.172847 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-67f858855d-4nzvz, hostIfaceName: a764270d8e23fad, network: default, NAD default, SandboxID: "a764270d8e23fad72578f31e519ca569fb215564f8493a714040778e74ec1061", PCI device ID: , UID: "54b9b208-6b4f-4eee-9af1-502c911768fb", MAC: 0a:58:0a:80:00:1b, IPs: [10.128.0.27/23] I0429 08:40:08.222269 3791 cni.go:473] [openshift-console/console-67f858855d-4nzvz a764270d8e23fad72578f31e519ca569fb215564f8493a714040778e74ec1061 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-67f858855d-4nzvz a764270d8e23fad72578f31e519ca569fb215564f8493a714040778e74ec1061 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:a3:42:27:e0:ab\",\"name\":\"a764270d8e23fad\"},{\"mac\":\"0a:58:0a:80:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d01c6a95-593d-48f6-a1e0-8fe27e214031\"}],\"ips\":[{\"address\":\"10.128.0.27/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:40:08.265453 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0429 08:40:08.265473 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0429 08:40:08.265511 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0429 08:40:09.270285 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-insights/metrics W0429 08:40:11.018893 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service openshift-console/downloads W0429 08:40:15.382439 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:40:15.382461 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:40:15.382493 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:40:15.382498 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:40:18.318499 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0429 08:40:22.328314 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0429 08:40:22.328368 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0429 08:40:22.328322 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics W0429 08:40:22.355643 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0429 08:40:22.355662 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0429 08:40:22.355698 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics W0429 08:40:23.342500 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0429 08:40:23.358256 3791 cni.go:452] [openshift-image-registry/image-registry-6d695d8795-7w94b 48a5a39d50f9bbd1869f1836bb954437f523fa5d5b26a2cdb040ff61ee3aa338 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-6d695d8795-7w94b 48a5a39d50f9bbd1869f1836bb954437f523fa5d5b26a2cdb040ff61ee3aa338 network default NAD default NAD key ] W0429 08:40:23.368911 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-service-ca-operator/metrics I0429 08:40:23.450751 3791 cni.go:473] [openshift-image-registry/image-registry-6d695d8795-7w94b 48a5a39d50f9bbd1869f1836bb954437f523fa5d5b26a2cdb040ff61ee3aa338 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-6d695d8795-7w94b 48a5a39d50f9bbd1869f1836bb954437f523fa5d5b26a2cdb040ff61ee3aa338 network default NAD default NAD key default], result "{}", err I0429 08:40:23.529291 3791 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6d695d8795-7w94b of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:40:23.529322 3791 pods.go:176] Deleting pod: openshift-image-registry/image-registry-6d695d8795-7w94b I0429 08:40:23.530665 3791 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-6d695d8795-7w94b, ips: 10.128.0.23 I0429 08:40:23.530695 3791 obj_retry.go:498] Detected object openshift-image-registry/image-registry-6d695d8795-7w94b of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:40:24.340078 3791 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6d695d8795-7w94b of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:40:24.340105 3791 obj_retry.go:492] Detected object openshift-image-registry/image-registry-6d695d8795-7w94b of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:40:24.350179 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-6d695d8795-7w94b I0429 08:40:24.350209 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-6d695d8795-7w94b I0429 08:40:33.191812 3791 cni.go:452] [openshift-ingress/router-default-675b7c69bc-txw8m 079bff5339c8c53e03730b543b6dc4925bed5290c348425a7e294b509816b022 network default NAD default NAD key ] DEL starting CNI request [openshift-ingress/router-default-675b7c69bc-txw8m 079bff5339c8c53e03730b543b6dc4925bed5290c348425a7e294b509816b022 network default NAD default NAD key ] I0429 08:40:33.275435 3791 cni.go:473] [openshift-ingress/router-default-675b7c69bc-txw8m 079bff5339c8c53e03730b543b6dc4925bed5290c348425a7e294b509816b022 network default NAD default NAD key default] DEL finished CNI request [openshift-ingress/router-default-675b7c69bc-txw8m 079bff5339c8c53e03730b543b6dc4925bed5290c348425a7e294b509816b022 network default NAD default NAD key default], result "{}", err I0429 08:40:33.326429 3791 obj_retry.go:498] Detected object openshift-ingress/router-default-675b7c69bc-txw8m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:40:33.326463 3791 pods.go:176] Deleting pod: openshift-ingress/router-default-675b7c69bc-txw8m I0429 08:40:33.327903 3791 pods.go:236] Attempting to release IPs for pod: openshift-ingress/router-default-675b7c69bc-txw8m, ips: 10.128.0.16 I0429 08:40:33.327937 3791 obj_retry.go:498] Detected object openshift-ingress/router-default-675b7c69bc-txw8m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:40:33.683401 3791 obj_retry.go:492] Detected object openshift-ingress/router-default-675b7c69bc-txw8m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:40:33.683422 3791 obj_retry.go:492] Detected object openshift-ingress/router-default-675b7c69bc-txw8m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:40:33.691253 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-675b7c69bc-txw8m I0429 08:40:33.691283 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-675b7c69bc-txw8m I0429 08:40:37.648477 3791 obj_retry.go:498] Detected object openshift-console/console-c49596959-5zb2m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:40:37.649089 3791 obj_retry.go:498] Detected object openshift-console/console-c49596959-5zb2m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:40:38.468119 3791 obj_retry.go:492] Detected object openshift-console/console-c49596959-5zb2m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:40:38.468140 3791 obj_retry.go:492] Detected object openshift-console/console-c49596959-5zb2m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:40:38.476341 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-c49596959-5zb2m I0429 08:40:38.476361 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-c49596959-5zb2m I0429 08:40:43.678100 3791 obj_retry.go:498] Detected object openshift-console/console-c49596959-w5fxl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:40:43.678622 3791 obj_retry.go:498] Detected object openshift-console/console-c49596959-w5fxl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:40:44.139499 3791 obj_retry.go:492] Detected object openshift-console/console-c49596959-w5fxl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:40:44.139526 3791 obj_retry.go:492] Detected object openshift-console/console-c49596959-w5fxl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:40:44.147722 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-c49596959-w5fxl I0429 08:40:44.147751 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-c49596959-w5fxl W0429 08:40:46.482418 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:40:46.482444 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0429 08:41:01.290207 3791 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:01.290742 3791 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:01.368511 3791 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0429 08:41:02.218412 3791 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:02.218440 3791 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:02.230525 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0429 08:41:02.230556 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0429 08:41:05.500893 3791 cni.go:452] [openshift-monitoring/prometheus-k8s-1 d60cf2fe3eb950e4c960515176840df86cd4a82e839c8599b689c238761b2090 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-1 d60cf2fe3eb950e4c960515176840df86cd4a82e839c8599b689c238761b2090 network default NAD default NAD key ] I0429 08:41:05.581979 3791 cni.go:473] [openshift-monitoring/prometheus-k8s-1 d60cf2fe3eb950e4c960515176840df86cd4a82e839c8599b689c238761b2090 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-1 d60cf2fe3eb950e4c960515176840df86cd4a82e839c8599b689c238761b2090 network default NAD default NAD key default], result "{}", err I0429 08:41:05.644316 3791 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:05.644359 3791 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-1 I0429 08:41:05.645778 3791 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-1, ips: 10.128.0.26 I0429 08:41:05.645811 3791 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:06.533542 3791 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:06.533560 3791 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:06.543531 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0429 08:41:06.543559 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0429 08:41:06.582377 3791 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-1] creating logical port openshift-monitoring_prometheus-k8s-1 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:41:06.582472 3791 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-1 I0429 08:41:06.610252 3791 pod.go:62] [openshift-monitoring/prometheus-k8s-1] pod update took 27.818564ms I0429 08:41:06.610278 3791 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-1] addLogicalPort annotation time took 27.847975ms I0429 08:41:06.611805 3791 pods.go:271] [openshift-monitoring/prometheus-k8s-1] addLogicalPort took 29.440778ms, libovsdb time 1.091249ms I0429 08:41:06.944667 3791 cni.go:452] [openshift-monitoring/prometheus-k8s-1 7cdd274ff0b9697c94f83aa2f9f23673d46f77235a6d13b8259f5861893a2a38 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-1 7cdd274ff0b9697c94f83aa2f9f23673d46f77235a6d13b8259f5861893a2a38 network default NAD default NAD key ] I0429 08:41:06.954639 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-1, hostIfaceName: 7cdd274ff0b9697, network: default, NAD default, SandboxID: "7cdd274ff0b9697c94f83aa2f9f23673d46f77235a6d13b8259f5861893a2a38", PCI device ID: , UID: "e1bab742-fa7a-4ced-ad55-957dbe6ea6f9", MAC: 0a:58:0a:80:00:1c, IPs: [10.128.0.28/23] I0429 08:41:07.015866 3791 cni.go:473] [openshift-monitoring/prometheus-k8s-1 7cdd274ff0b9697c94f83aa2f9f23673d46f77235a6d13b8259f5861893a2a38 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-1 7cdd274ff0b9697c94f83aa2f9f23673d46f77235a6d13b8259f5861893a2a38 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"6a:db:b2:1c:dd:ee\",\"name\":\"7cdd274ff0b9697\"},{\"mac\":\"0a:58:0a:80:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8460b45-4f55-496a-8b17-c25e316bee5e\"}],\"ips\":[{\"address\":\"10.128.0.28/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:41:26.473862 3791 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlc582 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:26.474415 3791 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlc582 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:26.474440 3791 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlc582 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:41:26.474448 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0429 08:41:26.474461 3791 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlc582 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:41:26.474466 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0429 08:41:27.501036 3791 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlc582 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:27.501063 3791 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlc582 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:27.501079 3791 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlc582 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:27.501091 3791 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0dlc582 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:30.131477 3791 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0429 08:41:30.143643 3791 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0429 08:41:30.528172 3791 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624200-hnwst of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:30.528735 3791 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624200-hnwst of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:31.378415 3791 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624200-hnwst of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:31.378439 3791 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624200-hnwst of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:33.946189 3791 obj_retry.go:498] Detected object openshift-console/console-67f858855d-8jljg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:33.946722 3791 obj_retry.go:498] Detected object openshift-console/console-67f858855d-8jljg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:34.034445 3791 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:34.034916 3791 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:34.683613 3791 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:34.683634 3791 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:34.692806 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0429 08:41:34.692846 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0429 08:41:34.715282 3791 obj_retry.go:492] Detected object openshift-console/console-67f858855d-8jljg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:34.715300 3791 obj_retry.go:492] Detected object openshift-console/console-67f858855d-8jljg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:34.725639 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-67f858855d-8jljg I0429 08:41:34.725659 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-67f858855d-8jljg I0429 08:41:44.516510 3791 cni.go:452] [openshift-console/console-67f858855d-4nzvz a764270d8e23fad72578f31e519ca569fb215564f8493a714040778e74ec1061 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-67f858855d-4nzvz a764270d8e23fad72578f31e519ca569fb215564f8493a714040778e74ec1061 network default NAD default NAD key ] I0429 08:41:44.585937 3791 cni.go:473] [openshift-console/console-67f858855d-4nzvz a764270d8e23fad72578f31e519ca569fb215564f8493a714040778e74ec1061 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-67f858855d-4nzvz a764270d8e23fad72578f31e519ca569fb215564f8493a714040778e74ec1061 network default NAD default NAD key default], result "{}", err I0429 08:41:44.632875 3791 obj_retry.go:498] Detected object openshift-console/console-67f858855d-4nzvz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:44.632901 3791 pods.go:176] Deleting pod: openshift-console/console-67f858855d-4nzvz I0429 08:41:44.634273 3791 pods.go:236] Attempting to release IPs for pod: openshift-console/console-67f858855d-4nzvz, ips: 10.128.0.27 I0429 08:41:44.634300 3791 obj_retry.go:498] Detected object openshift-console/console-67f858855d-4nzvz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:41:45.009028 3791 obj_retry.go:492] Detected object openshift-console/console-67f858855d-4nzvz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:45.009049 3791 obj_retry.go:492] Detected object openshift-console/console-67f858855d-4nzvz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:41:45.014128 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-67f858855d-4nzvz I0429 08:41:45.014150 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-67f858855d-4nzvz W0429 08:42:07.831074 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 08:42:07.831098 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0429 08:42:09.139958 3791 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:42:09.140543 3791 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:42:09.811056 3791 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:42:09.811085 3791 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:42:09.820197 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0429 08:42:09.820225 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0429 08:43:20.220553 3791 base_network_controller_pods.go:487] [default/openshift-console/console-74fd8f4ff6-b2qvr] creating logical port openshift-console_console-74fd8f4ff6-b2qvr for pod on switch ip-10-0-0-38.ec2.internal I0429 08:43:20.220705 3791 kube.go:256] Updating pod openshift-console/console-74fd8f4ff6-b2qvr I0429 08:43:20.261854 3791 pod.go:62] [openshift-console/console-74fd8f4ff6-b2qvr] pod update took 41.149127ms I0429 08:43:20.261885 3791 base_network_controller_pods.go:951] [default/openshift-console/console-74fd8f4ff6-b2qvr] addLogicalPort annotation time took 41.211428ms I0429 08:43:20.263280 3791 pods.go:271] [openshift-console/console-74fd8f4ff6-b2qvr] addLogicalPort took 42.744139ms, libovsdb time 927.772µs I0429 08:43:20.582820 3791 cni.go:452] [openshift-console/console-74fd8f4ff6-b2qvr 6828faa6593b82e2f48d154ae65e9cc53b341fef00b414eb344dbd48cf32465c network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-74fd8f4ff6-b2qvr 6828faa6593b82e2f48d154ae65e9cc53b341fef00b414eb344dbd48cf32465c network default NAD default NAD key ] I0429 08:43:20.591931 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-74fd8f4ff6-b2qvr, hostIfaceName: 6828faa6593b82e, network: default, NAD default, SandboxID: "6828faa6593b82e2f48d154ae65e9cc53b341fef00b414eb344dbd48cf32465c", PCI device ID: , UID: "52216fb6-95c3-4e80-b09d-b918480fb539", MAC: 0a:58:0a:80:00:1d, IPs: [10.128.0.29/23] I0429 08:43:20.640536 3791 cni.go:473] [openshift-console/console-74fd8f4ff6-b2qvr 6828faa6593b82e2f48d154ae65e9cc53b341fef00b414eb344dbd48cf32465c network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-74fd8f4ff6-b2qvr 6828faa6593b82e2f48d154ae65e9cc53b341fef00b414eb344dbd48cf32465c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:60:52:4d:a7:19\",\"name\":\"6828faa6593b82e\"},{\"mac\":\"0a:58:0a:80:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/c23648e8-198a-4838-84e6-36d716386021\"}],\"ips\":[{\"address\":\"10.128.0.29/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:43:31.125104 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0429 08:43:45.523291 3791 obj_retry.go:498] Detected object openshift-console/console-7d4f6d98cc-k7cwk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:43:45.523807 3791 obj_retry.go:498] Detected object openshift-console/console-7d4f6d98cc-k7cwk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:43:46.192280 3791 obj_retry.go:492] Detected object openshift-console/console-7d4f6d98cc-k7cwk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:43:46.192304 3791 obj_retry.go:492] Detected object openshift-console/console-7d4f6d98cc-k7cwk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:43:46.199919 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d4f6d98cc-k7cwk I0429 08:43:46.199942 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d4f6d98cc-k7cwk I0429 08:43:55.970441 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0429 08:43:56.468278 3791 obj_retry.go:498] Detected object openshift-console/console-7d4f6d98cc-rmpvc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:43:56.468797 3791 obj_retry.go:498] Detected object openshift-console/console-7d4f6d98cc-rmpvc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:43:56.918225 3791 obj_retry.go:492] Detected object openshift-console/console-7d4f6d98cc-rmpvc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:43:56.918247 3791 obj_retry.go:492] Detected object openshift-console/console-7d4f6d98cc-rmpvc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:43:56.926439 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-7d4f6d98cc-rmpvc I0429 08:43:56.926467 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-7d4f6d98cc-rmpvc I0429 08:44:06.961027 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=6 I0429 08:44:17.858144 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=85 I0429 08:44:33.987678 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=6 I0429 08:45:51.493752 3791 namespace.go:98] [tsf] adding namespace I0429 08:45:51.494658 3791 namespace.go:102] [tsf] adding namespace took 879.841µs I0429 08:45:51.514426 3791 namespace.go:142] [tsf] updating namespace I0429 08:45:51.535951 3791 namespace.go:142] [tsf] updating namespace I0429 08:45:56.087703 3791 namespace.go:98] [tsf-keycloak] adding namespace I0429 08:45:56.088636 3791 namespace.go:102] [tsf-keycloak] adding namespace took 906.681µs I0429 08:45:56.090268 3791 namespace.go:98] [tsf-tas] adding namespace I0429 08:45:56.090900 3791 namespace.go:102] [tsf-tas] adding namespace took 616.097µs I0429 08:45:56.092332 3791 namespace.go:98] [rhbk-operator] adding namespace I0429 08:45:56.092969 3791 namespace.go:102] [rhbk-operator] adding namespace took 621.433µs I0429 08:45:56.095147 3791 namespace.go:98] [rhtpa-operator] adding namespace I0429 08:45:56.095339 3791 namespace.go:98] [konflux-operator] adding namespace I0429 08:45:56.095763 3791 namespace.go:102] [rhtpa-operator] adding namespace took 599.18µs I0429 08:45:56.095785 3791 namespace.go:98] [cert-manager-operator] adding namespace I0429 08:45:56.096265 3791 namespace.go:102] [cert-manager-operator] adding namespace took 473.144µs I0429 08:45:56.096729 3791 namespace.go:102] [konflux-operator] adding namespace took 1.373647ms I0429 08:45:56.097810 3791 namespace.go:98] [konflux-ui] adding namespace I0429 08:45:56.098408 3791 namespace.go:102] [konflux-ui] adding namespace took 585.631µs I0429 08:45:56.102990 3791 namespace.go:98] [tsf-tpa] adding namespace I0429 08:45:56.103519 3791 namespace.go:102] [tsf-tpa] adding namespace took 516.719µs I0429 08:45:56.122884 3791 namespace.go:142] [tsf-keycloak] updating namespace I0429 08:45:56.140099 3791 namespace.go:142] [tsf-keycloak] updating namespace I0429 08:45:56.162819 3791 namespace.go:142] [tsf-tas] updating namespace I0429 08:45:56.169758 3791 namespace.go:142] [rhbk-operator] updating namespace I0429 08:45:56.172407 3791 namespace.go:142] [rhtpa-operator] updating namespace I0429 08:45:56.172443 3791 namespace.go:142] [cert-manager-operator] updating namespace I0429 08:45:56.185981 3791 namespace.go:142] [tsf-tas] updating namespace I0429 08:45:56.189139 3791 namespace.go:142] [konflux-operator] updating namespace I0429 08:45:56.195929 3791 namespace.go:142] [konflux-ui] updating namespace I0429 08:45:56.196006 3791 namespace.go:142] [tsf-tpa] updating namespace I0429 08:45:56.201711 3791 namespace.go:142] [tsf-keycloak] updating namespace I0429 08:45:56.262687 3791 namespace.go:142] [rhtpa-operator] updating namespace I0429 08:45:56.287642 3791 namespace.go:142] [rhtpa-operator] updating namespace I0429 08:45:56.287782 3791 namespace.go:142] [cert-manager-operator] updating namespace I0429 08:45:56.302149 3791 namespace.go:142] [tsf-tas] updating namespace I0429 08:45:56.302812 3791 namespace.go:142] [cert-manager-operator] updating namespace I0429 08:45:56.304739 3791 namespace.go:142] [konflux-operator] updating namespace I0429 08:45:56.336216 3791 namespace.go:142] [konflux-operator] updating namespace I0429 08:45:56.337155 3791 namespace.go:142] [konflux-ui] updating namespace I0429 08:45:56.373887 3791 namespace.go:142] [konflux-ui] updating namespace I0429 08:45:56.382821 3791 namespace.go:142] [tsf-tpa] updating namespace I0429 08:45:56.396286 3791 namespace.go:142] [tsf-tpa] updating namespace I0429 08:45:56.403954 3791 namespace.go:142] [rhbk-operator] updating namespace I0429 08:45:56.430865 3791 namespace.go:142] [rhbk-operator] updating namespace I0429 08:45:57.601400 3791 namespace.go:142] [tsf-tpa] updating namespace I0429 08:45:57.609023 3791 namespace.go:142] [tsf-keycloak] updating namespace I0429 08:45:59.976126 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=8 I0429 08:46:06.425158 3791 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7lmcb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:06.425749 3791 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7lmcb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:06.425774 3791 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7lmcb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:06.425783 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 08:46:06.425797 3791 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7lmcb of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:06.425801 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 08:46:06.452106 3791 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51vlhtv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:06.452582 3791 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51vlhtv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:06.452606 3791 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51vlhtv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:06.452616 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 08:46:06.452627 3791 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51vlhtv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:06.452631 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 08:46:07.449363 3791 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7lmcb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:07.449394 3791 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7lmcb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:07.449412 3791 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7lmcb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:07.449426 3791 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d7lmcb of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:07.477508 3791 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51vlhtv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:07.477523 3791 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51vlhtv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:07.477531 3791 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51vlhtv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:07.477538 3791 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51vlhtv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:11.617420 3791 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2c298d of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:11.617960 3791 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2c298d of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:11.617981 3791 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2c298d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:11.617990 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 08:46:11.618000 3791 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2c298d of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:11.618005 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 08:46:12.660283 3791 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158l4dmv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:12.660788 3791 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158l4dmv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:12.660846 3791 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158l4dmv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:12.660858 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 08:46:12.660871 3791 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158l4dmv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:12.660876 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 08:46:12.738143 3791 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7rhj6w of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:12.738568 3791 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7rhj6w of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:12.738590 3791 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7rhj6w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:12.738597 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 08:46:12.738607 3791 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7rhj6w of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:12.738611 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 08:46:12.769033 3791 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2c298d of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:12.769053 3791 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2c298d of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:12.769062 3791 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2c298d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:12.769073 3791 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2c298d of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:13.630077 3791 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136tgl5v of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:13.630566 3791 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136tgl5v of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:46:13.630587 3791 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136tgl5v of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:13.630594 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 08:46:13.630606 3791 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136tgl5v of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 08:46:13.630610 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 08:46:13.694399 3791 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158l4dmv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:13.694420 3791 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158l4dmv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:13.694440 3791 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158l4dmv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:13.694456 3791 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158l4dmv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:13.807358 3791 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7rhj6w of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:13.807375 3791 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7rhj6w of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:13.807384 3791 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7rhj6w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:13.807395 3791 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7rhj6w of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:14.674933 3791 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136tgl5v of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:14.674960 3791 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136tgl5v of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:14.674974 3791 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136tgl5v of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:14.674985 3791 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136tgl5v of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:46:15.982493 3791 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=8 W0429 08:46:19.841037 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0429 08:46:20.236079 3791 namespace.go:98] [cert-manager] adding namespace I0429 08:46:20.236965 3791 namespace.go:102] [cert-manager] adding namespace took 863.162µs I0429 08:46:20.311019 3791 namespace.go:142] [cert-manager] updating namespace I0429 08:46:20.344041 3791 namespace.go:142] [cert-manager] updating namespace W0429 08:46:20.906067 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0429 08:46:20.922613 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0429 08:46:21.237712 3791 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-dnqrm] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-dnqrm for pod on switch ip-10-0-0-38.ec2.internal I0429 08:46:21.237852 3791 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-dnqrm I0429 08:46:21.255190 3791 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-dnqrm] pod update took 17.384786ms I0429 08:46:21.255214 3791 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-dnqrm] addLogicalPort annotation time took 17.413231ms I0429 08:46:21.256414 3791 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-dnqrm] addLogicalPort took 18.720482ms, libovsdb time 849.732µs I0429 08:46:21.619731 3791 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-dnqrm 0cedf6e3d8f78a52bdeec2619fe5b49ed039c10387be4afb2099870d99387125 network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-dnqrm 0cedf6e3d8f78a52bdeec2619fe5b49ed039c10387be4afb2099870d99387125 network default NAD default NAD key ] I0429 08:46:21.628755 3791 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-dnqrm, hostIfaceName: 0cedf6e3d8f78a5, network: default, NAD default, SandboxID: "0cedf6e3d8f78a52bdeec2619fe5b49ed039c10387be4afb2099870d99387125", PCI device ID: , UID: "3c408f07-7062-46f4-8e6d-bd4b9a0ee538", MAC: 0a:58:0a:80:00:1e, IPs: [10.128.0.30/23] I0429 08:46:21.677739 3791 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-dnqrm 0cedf6e3d8f78a52bdeec2619fe5b49ed039c10387be4afb2099870d99387125 network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-dnqrm 0cedf6e3d8f78a52bdeec2619fe5b49ed039c10387be4afb2099870d99387125 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"be:38:2a:bc:1a:5b\",\"name\":\"0cedf6e3d8f78a5\"},{\"mac\":\"0a:58:0a:80:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e95c2c3c-8d71-4197-a0e5-d82791d75cc1\"}],\"ips\":[{\"address\":\"10.128.0.30/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:46:24.806674 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0429 08:46:24.806699 3791 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook I0429 08:46:26.206281 3791 namespace.go:142] [openshift-operators] updating namespace I0429 08:46:26.224688 3791 namespace.go:142] [openshift-operators] updating namespace I0429 08:46:28.858202 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=144 W0429 08:46:31.141198 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0429 08:46:31.141220 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0429 08:46:32.953030 3791 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0429 08:46:33.774199 3791 namespace.go:142] [openshift-operators] updating namespace I0429 08:46:34.569116 3791 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-plzvj] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-plzvj for pod on switch ip-10-0-0-38.ec2.internal I0429 08:46:34.569217 3791 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-plzvj I0429 08:46:34.659721 3791 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-plzvj] pod update took 90.52281ms I0429 08:46:34.659739 3791 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-plzvj] addLogicalPort annotation time took 90.546968ms I0429 08:46:34.660872 3791 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-plzvj] addLogicalPort took 91.774134ms, libovsdb time 800.168µs I0429 08:46:34.927533 3791 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-plzvj f833a9ae43eff784b9387b413c8107af82790f472f5afb04c7eeb497e1a363ef network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-plzvj f833a9ae43eff784b9387b413c8107af82790f472f5afb04c7eeb497e1a363ef network default NAD default NAD key ] I0429 08:46:34.936749 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-plzvj, hostIfaceName: f833a9ae43eff78, network: default, NAD default, SandboxID: "f833a9ae43eff784b9387b413c8107af82790f472f5afb04c7eeb497e1a363ef", PCI device ID: , UID: "a0343886-73fd-40eb-9715-dfda688587bc", MAC: 0a:58:0a:80:00:1f, IPs: [10.128.0.31/23] I0429 08:46:34.987477 3791 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-plzvj f833a9ae43eff784b9387b413c8107af82790f472f5afb04c7eeb497e1a363ef network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-plzvj f833a9ae43eff784b9387b413c8107af82790f472f5afb04c7eeb497e1a363ef network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c6:eb:d9:58:86:10\",\"name\":\"f833a9ae43eff78\"},{\"mac\":\"0a:58:0a:80:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/86b2486f-46c3-4f46-9c56-ca43341a99d1\"}],\"ips\":[{\"address\":\"10.128.0.31/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:46:37.146915 3791 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics W0429 08:46:38.837090 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0429 08:46:39.931678 3791 base_network_controller_pods.go:487] [default/rhbk-operator/rhbk-operator-fb86cd8d9-r5zq7] creating logical port rhbk-operator_rhbk-operator-fb86cd8d9-r5zq7 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:46:39.931796 3791 kube.go:256] Updating pod rhbk-operator/rhbk-operator-fb86cd8d9-r5zq7 I0429 08:46:39.988863 3791 pod.go:62] [rhbk-operator/rhbk-operator-fb86cd8d9-r5zq7] pod update took 57.095499ms I0429 08:46:39.988882 3791 base_network_controller_pods.go:951] [default/rhbk-operator/rhbk-operator-fb86cd8d9-r5zq7] addLogicalPort annotation time took 57.120816ms I0429 08:46:39.990050 3791 pods.go:271] [rhbk-operator/rhbk-operator-fb86cd8d9-r5zq7] addLogicalPort took 58.391704ms, libovsdb time 845.542µs I0429 08:46:40.290191 3791 cni.go:452] [rhbk-operator/rhbk-operator-fb86cd8d9-r5zq7 9e5c0af7ce9c7813adb2499773a882910ad50e4217c86c61511ac67464f02455 network default NAD default NAD key ] ADD starting CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-r5zq7 9e5c0af7ce9c7813adb2499773a882910ad50e4217c86c61511ac67464f02455 network default NAD default NAD key ] I0429 08:46:40.300119 3791 helper_linux.go:556] ConfigureOVS: namespace: rhbk-operator, podName: rhbk-operator-fb86cd8d9-r5zq7, hostIfaceName: 9e5c0af7ce9c781, network: default, NAD default, SandboxID: "9e5c0af7ce9c7813adb2499773a882910ad50e4217c86c61511ac67464f02455", PCI device ID: , UID: "45de11df-8703-4863-82d7-da41a28f0ae9", MAC: 0a:58:0a:80:00:20, IPs: [10.128.0.32/23] I0429 08:46:40.348043 3791 cni.go:473] [rhbk-operator/rhbk-operator-fb86cd8d9-r5zq7 9e5c0af7ce9c7813adb2499773a882910ad50e4217c86c61511ac67464f02455 network default NAD default NAD key default] ADD finished CNI request [rhbk-operator/rhbk-operator-fb86cd8d9-r5zq7 9e5c0af7ce9c7813adb2499773a882910ad50e4217c86c61511ac67464f02455 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"66:a6:a8:5d:eb:17\",\"name\":\"9e5c0af7ce9c781\"},{\"mac\":\"0a:58:0a:80:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/66333457-9ab0-43f9-adeb-69fd6f52747d\"}],\"ips\":[{\"address\":\"10.128.0.32/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:46:41.155216 3791 base_network_controller_pods.go:487] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-pwffz] creating logical port openshift-operators_rhtas-operator-controller-manager-68888cb75-pwffz for pod on switch ip-10-0-0-38.ec2.internal I0429 08:46:41.155385 3791 kube.go:256] Updating pod openshift-operators/rhtas-operator-controller-manager-68888cb75-pwffz I0429 08:46:41.201862 3791 pod.go:62] [openshift-operators/rhtas-operator-controller-manager-68888cb75-pwffz] pod update took 46.510227ms I0429 08:46:41.201890 3791 base_network_controller_pods.go:951] [default/openshift-operators/rhtas-operator-controller-manager-68888cb75-pwffz] addLogicalPort annotation time took 46.541148ms I0429 08:46:41.203310 3791 pods.go:271] [openshift-operators/rhtas-operator-controller-manager-68888cb75-pwffz] addLogicalPort took 48.112952ms, libovsdb time 1.035283ms I0429 08:46:41.521375 3791 cni.go:452] [openshift-operators/rhtas-operator-controller-manager-68888cb75-pwffz 997ec993de6fada6183940726c37b3887da08f6bcf065bc16bb3d3d629ba187c network default NAD default NAD key ] ADD starting CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-pwffz 997ec993de6fada6183940726c37b3887da08f6bcf065bc16bb3d3d629ba187c network default NAD default NAD key ] I0429 08:46:41.532889 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: rhtas-operator-controller-manager-68888cb75-pwffz, hostIfaceName: 997ec993de6fada, network: default, NAD default, SandboxID: "997ec993de6fada6183940726c37b3887da08f6bcf065bc16bb3d3d629ba187c", PCI device ID: , UID: "8b6099c8-9aac-4e5e-b19f-9398454bcd28", MAC: 0a:58:0a:80:00:21, IPs: [10.128.0.33/23] I0429 08:46:41.585190 3791 cni.go:473] [openshift-operators/rhtas-operator-controller-manager-68888cb75-pwffz 997ec993de6fada6183940726c37b3887da08f6bcf065bc16bb3d3d629ba187c network default NAD default NAD key default] ADD finished CNI request [openshift-operators/rhtas-operator-controller-manager-68888cb75-pwffz 997ec993de6fada6183940726c37b3887da08f6bcf065bc16bb3d3d629ba187c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:df:24:ad:da:24\",\"name\":\"997ec993de6fada\"},{\"mac\":\"0a:58:0a:80:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d6f2d69a-77a8-44dc-ba5a-989a651ff5a6\"}],\"ips\":[{\"address\":\"10.128.0.33/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:46:44.936119 3791 namespace.go:98] [trusted-artifact-signer] adding namespace I0429 08:46:44.936955 3791 namespace.go:102] [trusted-artifact-signer] adding namespace took 813.867µs W0429 08:46:45.031625 3791 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0429 08:46:45.041858 3791 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 08:46:45.109579 3791 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 08:46:48.037324 3791 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0429 08:46:48.037402 3791 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-vt6lk IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-vt6lk: no pod IPs found I0429 08:46:49.992634 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=10 I0429 08:46:50.481148 3791 namespace.go:98] [openshift-pipelines] adding namespace I0429 08:46:50.482056 3791 namespace.go:102] [openshift-pipelines] adding namespace took 886.894µs I0429 08:46:50.592333 3791 namespace.go:142] [openshift-pipelines] updating namespace I0429 08:46:51.442180 3791 namespace.go:142] [cert-manager] updating namespace I0429 08:46:51.655508 3791 namespace.go:142] [cert-manager-operator] updating namespace I0429 08:46:52.045747 3791 namespace.go:142] [konflux-operator] updating namespace I0429 08:46:52.247205 3791 namespace.go:142] [konflux-ui] updating namespace I0429 08:46:52.446740 3791 namespace.go:142] [open-cluster-management-2pvn461a2e3dg73kigr9ohvaqf1urjn8] updating namespace I0429 08:46:52.850925 3791 namespace.go:142] [rhbk-operator] updating namespace I0429 08:46:53.062406 3791 namespace.go:142] [rhtpa-operator] updating namespace I0429 08:46:53.252419 3791 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 08:46:53.452123 3791 namespace.go:142] [tsf] updating namespace I0429 08:46:53.652280 3791 namespace.go:142] [tsf-keycloak] updating namespace I0429 08:46:53.857970 3791 namespace.go:142] [tsf-tas] updating namespace I0429 08:46:54.053743 3791 namespace.go:142] [tsf-tpa] updating namespace I0429 08:46:54.205908 3791 namespace.go:142] [cert-manager] updating namespace I0429 08:46:54.218890 3791 namespace.go:142] [cert-manager-operator] updating namespace I0429 08:46:54.230235 3791 namespace.go:142] [default] updating namespace I0429 08:46:54.245853 3791 namespace.go:142] [konflux-operator] updating namespace I0429 08:46:54.261224 3791 namespace.go:142] [konflux-ui] updating namespace I0429 08:46:54.272378 3791 namespace.go:142] [open-cluster-management-2pvn461a2e3dg73kigr9ohvaqf1urjn8] updating namespace I0429 08:46:54.281873 3791 namespace.go:142] [openshift] updating namespace I0429 08:46:54.295313 3791 namespace.go:142] [rhbk-operator] updating namespace I0429 08:46:54.307051 3791 namespace.go:142] [rhtpa-operator] updating namespace I0429 08:46:54.318449 3791 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 08:46:54.334930 3791 namespace.go:142] [tsf] updating namespace I0429 08:46:54.347910 3791 namespace.go:142] [tsf-keycloak] updating namespace I0429 08:46:54.360451 3791 namespace.go:142] [tsf-tas] updating namespace I0429 08:46:54.372980 3791 namespace.go:142] [tsf-tpa] updating namespace I0429 08:46:54.443712 3791 namespace.go:142] [cert-manager] updating namespace I0429 08:46:54.551930 3791 namespace.go:142] [cert-manager-operator] updating namespace I0429 08:46:54.640121 3791 namespace.go:142] [default] updating namespace I0429 08:46:54.743236 3791 namespace.go:142] [konflux-operator] updating namespace I0429 08:46:54.841791 3791 namespace.go:142] [konflux-ui] updating namespace I0429 08:46:54.942570 3791 namespace.go:142] [open-cluster-management-2pvn461a2e3dg73kigr9ohvaqf1urjn8] updating namespace I0429 08:46:55.038894 3791 namespace.go:142] [openshift] updating namespace I0429 08:46:55.141674 3791 namespace.go:142] [rhbk-operator] updating namespace I0429 08:46:55.243817 3791 namespace.go:142] [rhtpa-operator] updating namespace I0429 08:46:55.339554 3791 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 08:46:55.440317 3791 namespace.go:142] [tsf] updating namespace I0429 08:46:55.544240 3791 namespace.go:142] [tsf-keycloak] updating namespace I0429 08:46:55.642001 3791 namespace.go:142] [tsf-tas] updating namespace I0429 08:46:55.746444 3791 namespace.go:142] [tsf-tpa] updating namespace W0429 08:46:55.913593 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service I0429 08:46:56.189789 3791 namespace.go:142] [openshift-pipelines] updating namespace I0429 08:47:01.907214 3791 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:47:01.907770 3791 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:47:01.933778 3791 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:47:01.933798 3791 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:47:01.943149 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0429 08:47:01.943167 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0429 08:47:03.944686 3791 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-p7b7s] creating logical port openshift-pipelines_tekton-pipelines-controller-86bd8d9bfc-p7b7s for pod on switch ip-10-0-0-38.ec2.internal I0429 08:47:03.944789 3791 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-p7b7s I0429 08:47:03.961808 3791 pod.go:62] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-p7b7s] pod update took 17.050562ms I0429 08:47:03.961895 3791 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-p7b7s] addLogicalPort annotation time took 17.079873ms I0429 08:47:03.963173 3791 pods.go:271] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-p7b7s] addLogicalPort took 18.50025ms, libovsdb time 829.54µs I0429 08:47:04.048244 3791 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-glqbg] creating logical port openshift-pipelines_tekton-events-controller-57d7f78b8d-glqbg for pod on switch ip-10-0-0-38.ec2.internal I0429 08:47:04.048366 3791 kube.go:256] Updating pod openshift-pipelines/tekton-events-controller-57d7f78b8d-glqbg I0429 08:47:04.062888 3791 pod.go:62] [openshift-pipelines/tekton-events-controller-57d7f78b8d-glqbg] pod update took 14.570601ms I0429 08:47:04.062908 3791 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-events-controller-57d7f78b8d-glqbg] addLogicalPort annotation time took 14.593108ms I0429 08:47:04.064064 3791 pods.go:271] [openshift-pipelines/tekton-events-controller-57d7f78b8d-glqbg] addLogicalPort took 15.847802ms, libovsdb time 812.52µs I0429 08:47:04.313137 3791 cni.go:452] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-p7b7s 2a891ff1886404d50bb1cabe5186dc7179d0971174867d330bcf154f055da03e network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-p7b7s 2a891ff1886404d50bb1cabe5186dc7179d0971174867d330bcf154f055da03e network default NAD default NAD key ] I0429 08:47:04.340725 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-controller-86bd8d9bfc-p7b7s, hostIfaceName: 2a891ff1886404d, network: default, NAD default, SandboxID: "2a891ff1886404d50bb1cabe5186dc7179d0971174867d330bcf154f055da03e", PCI device ID: , UID: "68239535-8364-4fb9-94a9-9c9093e480c9", MAC: 0a:58:0a:80:00:22, IPs: [10.128.0.34/23] I0429 08:47:04.391672 3791 cni.go:473] [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-p7b7s 2a891ff1886404d50bb1cabe5186dc7179d0971174867d330bcf154f055da03e network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-controller-86bd8d9bfc-p7b7s 2a891ff1886404d50bb1cabe5186dc7179d0971174867d330bcf154f055da03e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:ca:30:38:71:5f\",\"name\":\"2a891ff1886404d\"},{\"mac\":\"0a:58:0a:80:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6a6d3d6f-c142-44ae-a706-210243f20c25\"}],\"ips\":[{\"address\":\"10.128.0.34/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:47:04.412412 3791 cni.go:452] [openshift-pipelines/tekton-events-controller-57d7f78b8d-glqbg 550f34f9107040c58e2622a22a3aa81d0000f34344a21fc055701994200f14f2 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-glqbg 550f34f9107040c58e2622a22a3aa81d0000f34344a21fc055701994200f14f2 network default NAD default NAD key ] I0429 08:47:04.422066 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-events-controller-57d7f78b8d-glqbg, hostIfaceName: 550f34f9107040c, network: default, NAD default, SandboxID: "550f34f9107040c58e2622a22a3aa81d0000f34344a21fc055701994200f14f2", PCI device ID: , UID: "d55f4f2e-45e2-4f1c-aef8-3c99b3ce2f0a", MAC: 0a:58:0a:80:00:23, IPs: [10.128.0.35/23] I0429 08:47:04.471690 3791 cni.go:473] [openshift-pipelines/tekton-events-controller-57d7f78b8d-glqbg 550f34f9107040c58e2622a22a3aa81d0000f34344a21fc055701994200f14f2 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-events-controller-57d7f78b8d-glqbg 550f34f9107040c58e2622a22a3aa81d0000f34344a21fc055701994200f14f2 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:72:d2:a3:db:17\",\"name\":\"550f34f9107040c\"},{\"mac\":\"0a:58:0a:80:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/60ffdbe3-bf17-4065-a533-bb6922533de8\"}],\"ips\":[{\"address\":\"10.128.0.35/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:47:08.023913 3791 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 08:47:08.023936 3791 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 08:47:08.023942 3791 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 08:47:08.068872 3791 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 08:47:08.068890 3791 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 08:47:08.068894 3791 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 08:47:08.818375 3791 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 08:47:08.818397 3791 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 08:47:08.818403 3791 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 08:47:08.818407 3791 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 08:47:16.891511 3791 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql W0429 08:47:16.918480 3791 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0429 08:47:19.008464 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 08:47:19.008483 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 08:47:19.008488 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 08:47:19.031579 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 08:47:19.031597 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 08:47:19.031603 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0429 08:47:20.951014 3791 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=9 I0429 08:47:25.860435 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=105 I0429 08:47:25.860437 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=97 I0429 08:47:27.014995 3791 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-jcv5s] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-jcv5s for pod on switch ip-10-0-0-38.ec2.internal I0429 08:47:27.015075 3791 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-jcv5s I0429 08:47:27.030951 3791 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-jcv5s] pod update took 15.894265ms I0429 08:47:27.030974 3791 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-jcv5s] addLogicalPort annotation time took 15.922051ms I0429 08:47:27.032377 3791 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-jcv5s] addLogicalPort took 17.39755ms, libovsdb time 951.066µs I0429 08:47:27.385442 3791 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-jcv5s 679279525fa37e43d6714bab13b70d077278748fc2d302db139b3fcc9824e972 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-jcv5s 679279525fa37e43d6714bab13b70d077278748fc2d302db139b3fcc9824e972 network default NAD default NAD key ] I0429 08:47:27.394478 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-jcv5s, hostIfaceName: 679279525fa37e4, network: default, NAD default, SandboxID: "679279525fa37e43d6714bab13b70d077278748fc2d302db139b3fcc9824e972", PCI device ID: , UID: "551ac864-1fb1-477b-bf2c-acfea08ff2f8", MAC: 0a:58:0a:80:00:24, IPs: [10.128.0.36/23] I0429 08:47:27.444722 3791 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-jcv5s 679279525fa37e43d6714bab13b70d077278748fc2d302db139b3fcc9824e972 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-jcv5s 679279525fa37e43d6714bab13b70d077278748fc2d302db139b3fcc9824e972 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"72:3f:54:74:6c:af\",\"name\":\"679279525fa37e4\"},{\"mac\":\"0a:58:0a:80:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/225a944a-99c3-4dd5-a010-46f337e25b26\"}],\"ips\":[{\"address\":\"10.128.0.36/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:47:30.102062 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0429 08:47:31.866219 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=913 I0429 08:47:40.213189 3791 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:47:40.213733 3791 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:47:40.236342 3791 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:47:40.236360 3791 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:47:40.245807 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0429 08:47:40.245850 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0429 08:47:40.966135 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=9 W0429 08:47:41.124068 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0429 08:47:42.326106 3791 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0429 08:47:42.326839 3791 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0429 08:47:42.326864 3791 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0429 08:47:42.329309 3791 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel I0429 08:47:48.170275 3791 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0429 08:47:50.069116 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0429 08:47:50.069134 3791 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service W0429 08:47:50.070476 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0429 08:47:50.070491 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak I0429 08:47:52.956664 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 I0429 08:48:03.866072 3791 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-v5wht] creating logical port openshift-pipelines_tekton-results-watcher-7777c5dbd5-v5wht for pod on switch ip-10-0-0-38.ec2.internal I0429 08:48:03.866157 3791 kube.go:256] Updating pod openshift-pipelines/tekton-results-watcher-7777c5dbd5-v5wht I0429 08:48:03.882778 3791 pod.go:62] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-v5wht] pod update took 16.646274ms I0429 08:48:03.882801 3791 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-watcher-7777c5dbd5-v5wht] addLogicalPort annotation time took 16.672992ms I0429 08:48:03.884213 3791 pods.go:271] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-v5wht] addLogicalPort took 18.155124ms, libovsdb time 975.051µs I0429 08:48:04.233154 3791 cni.go:452] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-v5wht 48eaa5d0ff8e97a3cad69f252fac5dfd04fe102129e13a026882a61c5c3f65ae network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-v5wht 48eaa5d0ff8e97a3cad69f252fac5dfd04fe102129e13a026882a61c5c3f65ae network default NAD default NAD key ] I0429 08:48:04.242391 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-watcher-7777c5dbd5-v5wht, hostIfaceName: 48eaa5d0ff8e97a, network: default, NAD default, SandboxID: "48eaa5d0ff8e97a3cad69f252fac5dfd04fe102129e13a026882a61c5c3f65ae", PCI device ID: , UID: "f69a81d7-cd87-406a-bae4-19b124fa8783", MAC: 0a:58:0a:80:00:25, IPs: [10.128.0.37/23] I0429 08:48:04.292134 3791 cni.go:473] [openshift-pipelines/tekton-results-watcher-7777c5dbd5-v5wht 48eaa5d0ff8e97a3cad69f252fac5dfd04fe102129e13a026882a61c5c3f65ae network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-watcher-7777c5dbd5-v5wht 48eaa5d0ff8e97a3cad69f252fac5dfd04fe102129e13a026882a61c5c3f65ae network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3a:e8:fe:5f:ac:1f\",\"name\":\"48eaa5d0ff8e97a\"},{\"mac\":\"0a:58:0a:80:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/83086e6b-a979-456d-a805-8fe8bd114e2c\"}],\"ips\":[{\"address\":\"10.128.0.37/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:48:09.329412 3791 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0429 08:48:09.329436 3791 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0429 08:48:09.329441 3791 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0429 08:48:15.389769 3791 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:48:15.390311 3791 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:48:15.436660 3791 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:48:15.436680 3791 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:48:15.444718 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0429 08:48:15.444742 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam I0429 08:48:17.628181 3791 base_network_controller_pods.go:487] [default/tsf-tas/segment-backup-installation-l7mgr-796qq] creating logical port tsf-tas_segment-backup-installation-l7mgr-796qq for pod on switch ip-10-0-0-38.ec2.internal I0429 08:48:17.628285 3791 kube.go:256] Updating pod tsf-tas/segment-backup-installation-l7mgr-796qq I0429 08:48:17.738021 3791 pod.go:62] [tsf-tas/segment-backup-installation-l7mgr-796qq] pod update took 109.75643ms I0429 08:48:17.738045 3791 base_network_controller_pods.go:951] [default/tsf-tas/segment-backup-installation-l7mgr-796qq] addLogicalPort annotation time took 109.785487ms I0429 08:48:17.739229 3791 pods.go:271] [tsf-tas/segment-backup-installation-l7mgr-796qq] addLogicalPort took 111.067118ms, libovsdb time 806.229µs I0429 08:48:18.285479 3791 cni.go:452] [tsf-tas/segment-backup-installation-l7mgr-796qq 56dca4cf1cba324ef0bb31637b4f9151ab2bd5d6d0e4f669f3cf3319bf847fe1 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/segment-backup-installation-l7mgr-796qq 56dca4cf1cba324ef0bb31637b4f9151ab2bd5d6d0e4f669f3cf3319bf847fe1 network default NAD default NAD key ] I0429 08:48:18.294903 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: segment-backup-installation-l7mgr-796qq, hostIfaceName: 56dca4cf1cba324, network: default, NAD default, SandboxID: "56dca4cf1cba324ef0bb31637b4f9151ab2bd5d6d0e4f669f3cf3319bf847fe1", PCI device ID: , UID: "1e6f0894-bf24-4567-ace7-909865649ca3", MAC: 0a:58:0a:80:00:26, IPs: [10.128.0.38/23] I0429 08:48:18.349360 3791 cni.go:473] [tsf-tas/segment-backup-installation-l7mgr-796qq 56dca4cf1cba324ef0bb31637b4f9151ab2bd5d6d0e4f669f3cf3319bf847fe1 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/segment-backup-installation-l7mgr-796qq 56dca4cf1cba324ef0bb31637b4f9151ab2bd5d6d0e4f669f3cf3319bf847fe1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:47:d1:3b:6a:19\",\"name\":\"56dca4cf1cba324\"},{\"mac\":\"0a:58:0a:80:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2af98891-942a-47a7-b21b-fad398716fbd\"}],\"ips\":[{\"address\":\"10.128.0.38/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:48:18.650142 3791 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-5c6669cf47-gvz2d] creating logical port tsf-tas_fulcio-server-5c6669cf47-gvz2d for pod on switch ip-10-0-0-38.ec2.internal I0429 08:48:18.650238 3791 kube.go:256] Updating pod tsf-tas/fulcio-server-5c6669cf47-gvz2d I0429 08:48:18.669043 3791 pod.go:62] [tsf-tas/fulcio-server-5c6669cf47-gvz2d] pod update took 18.827868ms I0429 08:48:18.669065 3791 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-5c6669cf47-gvz2d] addLogicalPort annotation time took 18.855394ms I0429 08:48:18.670418 3791 pods.go:271] [tsf-tas/fulcio-server-5c6669cf47-gvz2d] addLogicalPort took 20.293908ms, libovsdb time 975.031µs I0429 08:48:19.015231 3791 cni.go:452] [tsf-tas/fulcio-server-5c6669cf47-gvz2d c002dd37a967a4383ece57f0642ba5c3b17281902b0cebd874020ef5df3b81d8 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-5c6669cf47-gvz2d c002dd37a967a4383ece57f0642ba5c3b17281902b0cebd874020ef5df3b81d8 network default NAD default NAD key ] I0429 08:48:19.024519 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-5c6669cf47-gvz2d, hostIfaceName: c002dd37a967a43, network: default, NAD default, SandboxID: "c002dd37a967a4383ece57f0642ba5c3b17281902b0cebd874020ef5df3b81d8", PCI device ID: , UID: "f5958de4-1e39-4bd7-ba47-5ea197aa2663", MAC: 0a:58:0a:80:00:27, IPs: [10.128.0.39/23] I0429 08:48:19.075562 3791 cni.go:473] [tsf-tas/fulcio-server-5c6669cf47-gvz2d c002dd37a967a4383ece57f0642ba5c3b17281902b0cebd874020ef5df3b81d8 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-5c6669cf47-gvz2d c002dd37a967a4383ece57f0642ba5c3b17281902b0cebd874020ef5df3b81d8 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:e2:b9:f8:67:99\",\"name\":\"c002dd37a967a43\"},{\"mac\":\"0a:58:0a:80:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a79d7573-f85d-4434-84e5-158c631363e0\"}],\"ips\":[{\"address\":\"10.128.0.39/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:48:19.130291 3791 base_network_controller_pods.go:487] [default/tsf-tas/rekor-createtree-job-khcdl-7ffq5] creating logical port tsf-tas_rekor-createtree-job-khcdl-7ffq5 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:48:19.130387 3791 kube.go:256] Updating pod tsf-tas/rekor-createtree-job-khcdl-7ffq5 I0429 08:48:19.152132 3791 pod.go:62] [tsf-tas/rekor-createtree-job-khcdl-7ffq5] pod update took 21.765232ms I0429 08:48:19.152165 3791 base_network_controller_pods.go:951] [default/tsf-tas/rekor-createtree-job-khcdl-7ffq5] addLogicalPort annotation time took 21.806008ms I0429 08:48:19.154142 3791 pods.go:271] [tsf-tas/rekor-createtree-job-khcdl-7ffq5] addLogicalPort took 23.867483ms, libovsdb time 1.398095ms I0429 08:48:19.285149 3791 base_network_controller_pods.go:487] [default/tsf-tas/trillian-logsigner-774cbcbddd-b775l] creating logical port tsf-tas_trillian-logsigner-774cbcbddd-b775l for pod on switch ip-10-0-0-38.ec2.internal I0429 08:48:19.285272 3791 kube.go:256] Updating pod tsf-tas/trillian-logsigner-774cbcbddd-b775l I0429 08:48:19.307255 3791 pod.go:62] [tsf-tas/trillian-logsigner-774cbcbddd-b775l] pod update took 22.009568ms I0429 08:48:19.307340 3791 base_network_controller_pods.go:951] [default/tsf-tas/trillian-logsigner-774cbcbddd-b775l] addLogicalPort annotation time took 22.101422ms I0429 08:48:19.309072 3791 pods.go:271] [tsf-tas/trillian-logsigner-774cbcbddd-b775l] addLogicalPort took 23.940753ms, libovsdb time 1.1562ms I0429 08:48:19.512937 3791 cni.go:452] [tsf-tas/rekor-createtree-job-khcdl-7ffq5 db5caa0d170b579d423c62aab5a780dffeb53f8e61448dc9068cadf4d8b36fe4 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-createtree-job-khcdl-7ffq5 db5caa0d170b579d423c62aab5a780dffeb53f8e61448dc9068cadf4d8b36fe4 network default NAD default NAD key ] I0429 08:48:19.524497 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-createtree-job-khcdl-7ffq5, hostIfaceName: db5caa0d170b579, network: default, NAD default, SandboxID: "db5caa0d170b579d423c62aab5a780dffeb53f8e61448dc9068cadf4d8b36fe4", PCI device ID: , UID: "7860e0da-afbe-49c4-954e-233087f163a0", MAC: 0a:58:0a:80:00:28, IPs: [10.128.0.40/23] I0429 08:48:19.589778 3791 cni.go:473] [tsf-tas/rekor-createtree-job-khcdl-7ffq5 db5caa0d170b579d423c62aab5a780dffeb53f8e61448dc9068cadf4d8b36fe4 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-createtree-job-khcdl-7ffq5 db5caa0d170b579d423c62aab5a780dffeb53f8e61448dc9068cadf4d8b36fe4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:79:b6:7e:54:e6\",\"name\":\"db5caa0d170b579\"},{\"mac\":\"0a:58:0a:80:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/ae1bf054-842c-4a71-90a0-268ddfb240ce\"}],\"ips\":[{\"address\":\"10.128.0.40/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:48:20.269767 3791 cni.go:452] [tsf-tas/trillian-logsigner-774cbcbddd-b775l 5cca654d9ec9833720c6243e68a3a87a2e6cc5f6383954f8d51ce363fbc7bbff network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-logsigner-774cbcbddd-b775l 5cca654d9ec9833720c6243e68a3a87a2e6cc5f6383954f8d51ce363fbc7bbff network default NAD default NAD key ] I0429 08:48:20.283493 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-logsigner-774cbcbddd-b775l, hostIfaceName: 5cca654d9ec9833, network: default, NAD default, SandboxID: "5cca654d9ec9833720c6243e68a3a87a2e6cc5f6383954f8d51ce363fbc7bbff", PCI device ID: , UID: "f324966d-4f08-481f-b060-358f07d04a5a", MAC: 0a:58:0a:80:00:29, IPs: [10.128.0.41/23] I0429 08:48:20.362113 3791 cni.go:473] [tsf-tas/trillian-logsigner-774cbcbddd-b775l 5cca654d9ec9833720c6243e68a3a87a2e6cc5f6383954f8d51ce363fbc7bbff network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-logsigner-774cbcbddd-b775l 5cca654d9ec9833720c6243e68a3a87a2e6cc5f6383954f8d51ce363fbc7bbff network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:1f:bf:4f:57:39\",\"name\":\"5cca654d9ec9833\"},{\"mac\":\"0a:58:0a:80:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/20ffc190-d7a0-43f1-85a6-67218c6eac6e\"}],\"ips\":[{\"address\":\"10.128.0.41/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:48:21.903241 3791 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-775d4b5bc5-5lqjr] creating logical port tsf-tas_trillian-db-775d4b5bc5-5lqjr for pod on switch ip-10-0-0-38.ec2.internal I0429 08:48:21.903435 3791 kube.go:256] Updating pod tsf-tas/trillian-db-775d4b5bc5-5lqjr I0429 08:48:21.916573 3791 pod.go:62] [tsf-tas/trillian-db-775d4b5bc5-5lqjr] pod update took 13.172419ms I0429 08:48:21.916597 3791 base_network_controller_pods.go:951] [default/tsf-tas/trillian-db-775d4b5bc5-5lqjr] addLogicalPort annotation time took 13.203168ms I0429 08:48:21.918233 3791 pods.go:271] [tsf-tas/trillian-db-775d4b5bc5-5lqjr] addLogicalPort took 15.01384ms, libovsdb time 1.046048ms W0429 08:48:26.428492 3791 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0429 08:48:27.857802 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=10 I0429 08:48:31.509331 3791 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-tktsl of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:48:31.509978 3791 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-tktsl of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:48:31.577077 3791 cni.go:452] [tsf-tas/trillian-db-775d4b5bc5-5lqjr 60b4e3a1785dcee722d0ca1d9b31cdb664fb7dcf6033f567ceecaf7053a19f8f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-db-775d4b5bc5-5lqjr 60b4e3a1785dcee722d0ca1d9b31cdb664fb7dcf6033f567ceecaf7053a19f8f network default NAD default NAD key ] I0429 08:48:31.586400 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-775d4b5bc5-5lqjr, hostIfaceName: 60b4e3a1785dcee, network: default, NAD default, SandboxID: "60b4e3a1785dcee722d0ca1d9b31cdb664fb7dcf6033f567ceecaf7053a19f8f", PCI device ID: , UID: "71cc4d0b-6da7-4069-a024-3de005640786", MAC: 0a:58:0a:80:00:2a, IPs: [10.128.0.42/23] I0429 08:48:31.642780 3791 cni.go:473] [tsf-tas/trillian-db-775d4b5bc5-5lqjr 60b4e3a1785dcee722d0ca1d9b31cdb664fb7dcf6033f567ceecaf7053a19f8f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-db-775d4b5bc5-5lqjr 60b4e3a1785dcee722d0ca1d9b31cdb664fb7dcf6033f567ceecaf7053a19f8f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:ac:b9:3a:93:aa\",\"name\":\"60b4e3a1785dcee\"},{\"mac\":\"0a:58:0a:80:00:2a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/9523af2c-dc7a-4858-8974-cfbede5a2d6d\"}],\"ips\":[{\"address\":\"10.128.0.42/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:48:32.391467 3791 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-tktsl of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:48:32.391501 3791 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-tktsl of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 08:48:32.486050 3791 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner I0429 08:48:35.469949 3791 cni.go:452] [tsf-tas/segment-backup-installation-l7mgr-796qq 56dca4cf1cba324ef0bb31637b4f9151ab2bd5d6d0e4f669f3cf3319bf847fe1 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/segment-backup-installation-l7mgr-796qq 56dca4cf1cba324ef0bb31637b4f9151ab2bd5d6d0e4f669f3cf3319bf847fe1 network default NAD default NAD key ] I0429 08:48:35.553252 3791 cni.go:473] [tsf-tas/segment-backup-installation-l7mgr-796qq 56dca4cf1cba324ef0bb31637b4f9151ab2bd5d6d0e4f669f3cf3319bf847fe1 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/segment-backup-installation-l7mgr-796qq 56dca4cf1cba324ef0bb31637b4f9151ab2bd5d6d0e4f669f3cf3319bf847fe1 network default NAD default NAD key default], result "{}", err I0429 08:48:35.618535 3791 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-l7mgr-796qq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:48:35.618571 3791 pods.go:176] Deleting pod: tsf-tas/segment-backup-installation-l7mgr-796qq I0429 08:48:35.619867 3791 pods.go:236] Attempting to release IPs for pod: tsf-tas/segment-backup-installation-l7mgr-796qq, ips: 10.128.0.38 I0429 08:48:35.619897 3791 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-l7mgr-796qq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:48:36.635651 3791 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-l7mgr-796qq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:48:36.635679 3791 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-l7mgr-796qq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 08:48:40.499148 3791 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0429 08:48:42.435049 3791 util.go:870] Failed to build global endpoints for port TCP/http-listener: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0429 08:48:42.435066 3791 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-controller W0429 08:48:44.448985 3791 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0429 08:48:51.546365 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql W0429 08:49:09.596385 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8091: empty IP address endpoints for service tsf-tas/trillian-logsigner I0429 08:49:14.762090 3791 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:14.762657 3791 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:14.783727 3791 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:14.783742 3791 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:14.791301 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0429 08:49:14.791325 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0429 08:49:26.827180 3791 obj_retry.go:498] Detected object tsf/patch-tekton-config-t7nbf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:26.827715 3791 obj_retry.go:498] Detected object tsf/patch-tekton-config-t7nbf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:27.710204 3791 obj_retry.go:492] Detected object tsf/patch-tekton-config-t7nbf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:27.710227 3791 obj_retry.go:492] Detected object tsf/patch-tekton-config-t7nbf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:35.878119 3791 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-l22lj of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:35.878572 3791 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-l22lj of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:36.900552 3791 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-l22lj of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:36.900580 3791 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-l22lj of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:36.935165 3791 obj_retry.go:492] Detected object tsf/patch-tekton-config-t7nbf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:36.935191 3791 obj_retry.go:492] Detected object tsf/patch-tekton-config-t7nbf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:36.939444 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-t7nbf I0429 08:49:36.939473 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-t7nbf I0429 08:49:40.929279 3791 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:40.929792 3791 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:40.947140 3791 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:40.947157 3791 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:40.958020 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0429 08:49:40.958046 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0429 08:49:41.953184 3791 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-jf5x8-hpwsq of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:41.953719 3791 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-jf5x8-hpwsq of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:42.822285 3791 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-jf5x8-hpwsq of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:42.822305 3791 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-jf5x8-hpwsq of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 08:49:46.842058 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0429 08:49:46.989254 3791 obj_retry.go:498] Detected object tsf-tpa/create-db-js87p of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:46.989766 3791 obj_retry.go:498] Detected object tsf-tpa/create-db-js87p of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:48.015276 3791 obj_retry.go:492] Detected object tsf-tpa/create-db-js87p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:48.015301 3791 obj_retry.go:492] Detected object tsf-tpa/create-db-js87p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:51.014489 3791 obj_retry.go:498] Detected object tsf-tpa/migrate-db-tdxfr of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:51.015086 3791 obj_retry.go:498] Detected object tsf-tpa/migrate-db-tdxfr of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:51.178115 3791 obj_retry.go:492] Detected object tsf-tpa/migrate-db-tdxfr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:51.178137 3791 obj_retry.go:492] Detected object tsf-tpa/migrate-db-tdxfr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:51.400124 3791 base_network_controller_pods.go:487] [default/tsf-tpa/create-importers-j4z9z] creating logical port tsf-tpa_create-importers-j4z9z for pod on switch ip-10-0-0-38.ec2.internal I0429 08:49:51.400227 3791 kube.go:256] Updating pod tsf-tpa/create-importers-j4z9z I0429 08:49:51.415162 3791 pod.go:62] [tsf-tpa/create-importers-j4z9z] pod update took 14.961484ms I0429 08:49:51.415187 3791 base_network_controller_pods.go:951] [default/tsf-tpa/create-importers-j4z9z] addLogicalPort annotation time took 14.98857ms I0429 08:49:51.416476 3791 pods.go:271] [tsf-tpa/create-importers-j4z9z] addLogicalPort took 16.372954ms, libovsdb time 892.07µs I0429 08:49:51.759575 3791 cni.go:452] [tsf-tpa/create-importers-j4z9z 4d4711d83f0bf5e2e8922e528558235d7596290fe397a8abc9f908ccdb8dc2c6 network default NAD default NAD key ] ADD starting CNI request [tsf-tpa/create-importers-j4z9z 4d4711d83f0bf5e2e8922e528558235d7596290fe397a8abc9f908ccdb8dc2c6 network default NAD default NAD key ] I0429 08:49:51.768686 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tpa, podName: create-importers-j4z9z, hostIfaceName: 4d4711d83f0bf5e, network: default, NAD default, SandboxID: "4d4711d83f0bf5e2e8922e528558235d7596290fe397a8abc9f908ccdb8dc2c6", PCI device ID: , UID: "d6a19ffc-293f-4d2e-9ed5-578e7ddc6cdf", MAC: 0a:58:0a:80:00:2b, IPs: [10.128.0.43/23] I0429 08:49:51.820926 3791 cni.go:473] [tsf-tpa/create-importers-j4z9z 4d4711d83f0bf5e2e8922e528558235d7596290fe397a8abc9f908ccdb8dc2c6 network default NAD default NAD key default] ADD finished CNI request [tsf-tpa/create-importers-j4z9z 4d4711d83f0bf5e2e8922e528558235d7596290fe397a8abc9f908ccdb8dc2c6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:ea:18:b2:7e:e8\",\"name\":\"4d4711d83f0bf5e\"},{\"mac\":\"0a:58:0a:80:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/034ec0d5-185b-4be0-a234-466c1ec02af3\"}],\"ips\":[{\"address\":\"10.128.0.43/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:49:52.857863 3791 cni.go:452] [tsf-tas/rekor-createtree-job-khcdl-7ffq5 db5caa0d170b579d423c62aab5a780dffeb53f8e61448dc9068cadf4d8b36fe4 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/rekor-createtree-job-khcdl-7ffq5 db5caa0d170b579d423c62aab5a780dffeb53f8e61448dc9068cadf4d8b36fe4 network default NAD default NAD key ] I0429 08:49:52.942006 3791 cni.go:473] [tsf-tas/rekor-createtree-job-khcdl-7ffq5 db5caa0d170b579d423c62aab5a780dffeb53f8e61448dc9068cadf4d8b36fe4 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/rekor-createtree-job-khcdl-7ffq5 db5caa0d170b579d423c62aab5a780dffeb53f8e61448dc9068cadf4d8b36fe4 network default NAD default NAD key default], result "{}", err I0429 08:49:52.993344 3791 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-khcdl-7ffq5 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:52.993445 3791 pods.go:176] Deleting pod: tsf-tas/rekor-createtree-job-khcdl-7ffq5 I0429 08:49:52.994906 3791 pods.go:236] Attempting to release IPs for pod: tsf-tas/rekor-createtree-job-khcdl-7ffq5, ips: 10.128.0.40 I0429 08:49:52.994936 3791 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-khcdl-7ffq5 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:53.442118 3791 namespace.go:142] [openshift-pipelines] updating namespace I0429 08:49:53.878991 3791 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-khcdl-7ffq5 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:53.879019 3791 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-khcdl-7ffq5 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:55.878573 3791 cni.go:452] [tsf-tpa/create-importers-j4z9z 4d4711d83f0bf5e2e8922e528558235d7596290fe397a8abc9f908ccdb8dc2c6 network default NAD default NAD key ] DEL starting CNI request [tsf-tpa/create-importers-j4z9z 4d4711d83f0bf5e2e8922e528558235d7596290fe397a8abc9f908ccdb8dc2c6 network default NAD default NAD key ] I0429 08:49:55.953060 3791 cni.go:473] [tsf-tpa/create-importers-j4z9z 4d4711d83f0bf5e2e8922e528558235d7596290fe397a8abc9f908ccdb8dc2c6 network default NAD default NAD key default] DEL finished CNI request [tsf-tpa/create-importers-j4z9z 4d4711d83f0bf5e2e8922e528558235d7596290fe397a8abc9f908ccdb8dc2c6 network default NAD default NAD key default], result "{}", err I0429 08:49:55.979109 3791 base_network_controller_pods.go:487] [default/tsf-tas/rekor-redis-57ddf695d5-p76vc] creating logical port tsf-tas_rekor-redis-57ddf695d5-p76vc for pod on switch ip-10-0-0-38.ec2.internal I0429 08:49:55.979197 3791 kube.go:256] Updating pod tsf-tas/rekor-redis-57ddf695d5-p76vc I0429 08:49:55.994343 3791 pod.go:62] [tsf-tas/rekor-redis-57ddf695d5-p76vc] pod update took 15.169877ms I0429 08:49:55.994367 3791 base_network_controller_pods.go:951] [default/tsf-tas/rekor-redis-57ddf695d5-p76vc] addLogicalPort annotation time took 15.199704ms I0429 08:49:55.995624 3791 pods.go:271] [tsf-tas/rekor-redis-57ddf695d5-p76vc] addLogicalPort took 16.52835ms, libovsdb time 881.123µs I0429 08:49:56.005199 3791 obj_retry.go:498] Detected object tsf-tpa/create-importers-j4z9z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:56.005232 3791 pods.go:176] Deleting pod: tsf-tpa/create-importers-j4z9z I0429 08:49:56.006608 3791 pods.go:236] Attempting to release IPs for pod: tsf-tpa/create-importers-j4z9z, ips: 10.128.0.43 I0429 08:49:56.006640 3791 obj_retry.go:498] Detected object tsf-tpa/create-importers-j4z9z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:49:56.941674 3791 cni.go:452] [tsf-tas/rekor-redis-57ddf695d5-p76vc 4b472f3a927f1316b226321ccedf525c0683195baa9a33412c08034159d933bc network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-redis-57ddf695d5-p76vc 4b472f3a927f1316b226321ccedf525c0683195baa9a33412c08034159d933bc network default NAD default NAD key ] I0429 08:49:56.960206 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-redis-57ddf695d5-p76vc, hostIfaceName: 4b472f3a927f131, network: default, NAD default, SandboxID: "4b472f3a927f1316b226321ccedf525c0683195baa9a33412c08034159d933bc", PCI device ID: , UID: "833f0835-4c2b-4fbd-afff-532d9196ad76", MAC: 0a:58:0a:80:00:2c, IPs: [10.128.0.44/23] I0429 08:49:57.013192 3791 cni.go:473] [tsf-tas/rekor-redis-57ddf695d5-p76vc 4b472f3a927f1316b226321ccedf525c0683195baa9a33412c08034159d933bc network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-redis-57ddf695d5-p76vc 4b472f3a927f1316b226321ccedf525c0683195baa9a33412c08034159d933bc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"c2:fd:95:30:8f:08\",\"name\":\"4b472f3a927f131\"},{\"mac\":\"0a:58:0a:80:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b8952e72-b339-4322-87c1-e9e4092095d0\"}],\"ips\":[{\"address\":\"10.128.0.44/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:49:57.027593 3791 obj_retry.go:492] Detected object tsf-tpa/create-importers-j4z9z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:57.027616 3791 obj_retry.go:492] Detected object tsf-tpa/create-importers-j4z9z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:49:58.593118 3791 base_network_controller_pods.go:487] [default/tsf-tas/rekor-server-74c6fddb4d-nvlsp] creating logical port tsf-tas_rekor-server-74c6fddb4d-nvlsp for pod on switch ip-10-0-0-38.ec2.internal I0429 08:49:58.593208 3791 kube.go:256] Updating pod tsf-tas/rekor-server-74c6fddb4d-nvlsp I0429 08:49:58.606316 3791 pod.go:62] [tsf-tas/rekor-server-74c6fddb4d-nvlsp] pod update took 13.127631ms I0429 08:49:58.606344 3791 base_network_controller_pods.go:951] [default/tsf-tas/rekor-server-74c6fddb4d-nvlsp] addLogicalPort annotation time took 13.162493ms I0429 08:49:58.607636 3791 pods.go:271] [tsf-tas/rekor-server-74c6fddb4d-nvlsp] addLogicalPort took 14.536269ms, libovsdb time 862.411µs W0429 08:49:58.892769 3791 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis W0429 08:49:59.926260 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0429 08:50:00.171975 3791 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29624210-rgm6l for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:00.172135 3791 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l I0429 08:50:00.185196 3791 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l] pod update took 13.092036ms I0429 08:50:00.185222 3791 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l] addLogicalPort annotation time took 13.125173ms I0429 08:50:00.186473 3791 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l] addLogicalPort took 14.523051ms, libovsdb time 861.839µs I0429 08:50:00.534396 3791 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l 8af6f4759c5e8ecf2593adc983e66508a68a2417143339c127b5a27d54a7d42e network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l 8af6f4759c5e8ecf2593adc983e66508a68a2417143339c127b5a27d54a7d42e network default NAD default NAD key ] I0429 08:50:00.544208 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29624210-rgm6l, hostIfaceName: 8af6f4759c5e8ec, network: default, NAD default, SandboxID: "8af6f4759c5e8ecf2593adc983e66508a68a2417143339c127b5a27d54a7d42e", PCI device ID: , UID: "b169f6be-2c67-4a2b-a71a-56cdf7b3f468", MAC: 0a:58:0a:80:00:2e, IPs: [10.128.0.46/23] I0429 08:50:00.595620 3791 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l 8af6f4759c5e8ecf2593adc983e66508a68a2417143339c127b5a27d54a7d42e network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l 8af6f4759c5e8ecf2593adc983e66508a68a2417143339c127b5a27d54a7d42e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e6:a2:95:53:1b:d4\",\"name\":\"8af6f4759c5e8ec\"},{\"mac\":\"0a:58:0a:80:00:2e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5c3aa03e-513c-4d79-92c2-b0287a9cec8e\"}],\"ips\":[{\"address\":\"10.128.0.46/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:01.655531 3791 cni.go:452] [tsf-tas/rekor-server-74c6fddb4d-nvlsp cc8ef96142f4cb8ea7caeae54ca0ca964dd5e23f0ed136ab54f8094f9802029d network default NAD default NAD key ] ADD starting CNI request [tsf-tas/rekor-server-74c6fddb4d-nvlsp cc8ef96142f4cb8ea7caeae54ca0ca964dd5e23f0ed136ab54f8094f9802029d network default NAD default NAD key ] I0429 08:50:01.665566 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: rekor-server-74c6fddb4d-nvlsp, hostIfaceName: cc8ef96142f4cb8, network: default, NAD default, SandboxID: "cc8ef96142f4cb8ea7caeae54ca0ca964dd5e23f0ed136ab54f8094f9802029d", PCI device ID: , UID: "300d4c0a-b487-46a0-96fa-64ee638ed60a", MAC: 0a:58:0a:80:00:2d, IPs: [10.128.0.45/23] I0429 08:50:01.716922 3791 cni.go:473] [tsf-tas/rekor-server-74c6fddb4d-nvlsp cc8ef96142f4cb8ea7caeae54ca0ca964dd5e23f0ed136ab54f8094f9802029d network default NAD default NAD key default] ADD finished CNI request [tsf-tas/rekor-server-74c6fddb4d-nvlsp cc8ef96142f4cb8ea7caeae54ca0ca964dd5e23f0ed136ab54f8094f9802029d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:5f:f6:0a:fb:a1\",\"name\":\"cc8ef96142f4cb8\"},{\"mac\":\"0a:58:0a:80:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1c801a4c-3f07-4b14-b0e5-bf9d4c28fbdd\"}],\"ips\":[{\"address\":\"10.128.0.45/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:03.931670 3791 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l 8af6f4759c5e8ecf2593adc983e66508a68a2417143339c127b5a27d54a7d42e network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l 8af6f4759c5e8ecf2593adc983e66508a68a2417143339c127b5a27d54a7d42e network default NAD default NAD key ] I0429 08:50:04.016246 3791 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l 8af6f4759c5e8ecf2593adc983e66508a68a2417143339c127b5a27d54a7d42e network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l 8af6f4759c5e8ecf2593adc983e66508a68a2417143339c127b5a27d54a7d42e network default NAD default NAD key default], result "{}", err I0429 08:50:04.067110 3791 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:50:04.067156 3791 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l I0429 08:50:04.068696 3791 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l, ips: 10.128.0.46 I0429 08:50:04.068727 3791 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:50:04.941262 3791 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:50:04.941288 3791 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624210-rgm6l of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 08:50:05.933336 3791 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server W0429 08:50:05.939724 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server W0429 08:50:10.938224 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 6379: empty IP address endpoints for service tsf-tas/rekor-redis W0429 08:50:16.951690 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 3000: empty IP address endpoints for service tsf-tas/rekor-server I0429 08:50:21.918416 3791 base_network_controller_pods.go:487] [default/tsf-tas/tuf-repository-init-2tl66-xvrfh] creating logical port tsf-tas_tuf-repository-init-2tl66-xvrfh for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:21.918544 3791 kube.go:256] Updating pod tsf-tas/tuf-repository-init-2tl66-xvrfh I0429 08:50:21.933075 3791 pod.go:62] [tsf-tas/tuf-repository-init-2tl66-xvrfh] pod update took 14.555013ms I0429 08:50:21.933098 3791 base_network_controller_pods.go:951] [default/tsf-tas/tuf-repository-init-2tl66-xvrfh] addLogicalPort annotation time took 14.583755ms I0429 08:50:21.934542 3791 pods.go:271] [tsf-tas/tuf-repository-init-2tl66-xvrfh] addLogicalPort took 16.143588ms, libovsdb time 1.024425ms I0429 08:50:32.480118 3791 cni.go:452] [tsf-tas/tuf-repository-init-2tl66-xvrfh 3e5b12765cf242ea4455f7a2790443371022684848a9dcd38fbab8ff20971320 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-repository-init-2tl66-xvrfh 3e5b12765cf242ea4455f7a2790443371022684848a9dcd38fbab8ff20971320 network default NAD default NAD key ] I0429 08:50:32.489511 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-repository-init-2tl66-xvrfh, hostIfaceName: 3e5b12765cf242e, network: default, NAD default, SandboxID: "3e5b12765cf242ea4455f7a2790443371022684848a9dcd38fbab8ff20971320", PCI device ID: , UID: "2ee8fd9c-7d61-41ae-8009-c57d7a5f2231", MAC: 0a:58:0a:80:00:2f, IPs: [10.128.0.47/23] I0429 08:50:32.540104 3791 cni.go:473] [tsf-tas/tuf-repository-init-2tl66-xvrfh 3e5b12765cf242ea4455f7a2790443371022684848a9dcd38fbab8ff20971320 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-repository-init-2tl66-xvrfh 3e5b12765cf242ea4455f7a2790443371022684848a9dcd38fbab8ff20971320 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7e:60:82:a5:11:92\",\"name\":\"3e5b12765cf242e\"},{\"mac\":\"0a:58:0a:80:00:2f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/26608429-6a40-4605-92c2-df7ea70603f0\"}],\"ips\":[{\"address\":\"10.128.0.47/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:34.239354 3791 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:50:34.239909 3791 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:50:34.285203 3791 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:50:34.285225 3791 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:50:34.291440 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0429 08:50:34.291460 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0429 08:50:35.151413 3791 namespace.go:98] [build-service] adding namespace I0429 08:50:35.152435 3791 namespace.go:102] [build-service] adding namespace took 1.002006ms I0429 08:50:35.157156 3791 namespace.go:98] [default-tenant] adding namespace I0429 08:50:35.157943 3791 namespace.go:102] [default-tenant] adding namespace took 769.49µs I0429 08:50:35.157979 3791 namespace.go:98] [image-controller] adding namespace I0429 08:50:35.158625 3791 namespace.go:102] [image-controller] adding namespace took 636.228µs I0429 08:50:35.159411 3791 namespace.go:98] [integration-service] adding namespace I0429 08:50:35.160125 3791 namespace.go:102] [integration-service] adding namespace took 700.949µs I0429 08:50:35.251143 3791 namespace.go:142] [build-service] updating namespace I0429 08:50:35.283391 3791 namespace.go:142] [build-service] updating namespace I0429 08:50:35.295489 3791 namespace.go:142] [default-tenant] updating namespace I0429 08:50:35.322694 3791 namespace.go:142] [default-tenant] updating namespace I0429 08:50:35.339178 3791 namespace.go:142] [image-controller] updating namespace I0429 08:50:35.398206 3791 namespace.go:142] [image-controller] updating namespace I0429 08:50:35.398233 3791 namespace.go:142] [build-service] updating namespace I0429 08:50:35.405865 3791 namespace.go:142] [integration-service] updating namespace I0429 08:50:35.431414 3791 namespace.go:142] [integration-service] updating namespace I0429 08:50:35.465750 3791 namespace.go:142] [integration-service] updating namespace I0429 08:50:35.467961 3791 namespace.go:98] [release-service] adding namespace I0429 08:50:35.468765 3791 namespace.go:102] [release-service] adding namespace took 787.99µs I0429 08:50:35.508304 3791 namespace.go:142] [konflux-ui] updating namespace I0429 08:50:35.523379 3791 namespace.go:142] [release-service] updating namespace I0429 08:50:35.559976 3791 namespace.go:142] [release-service] updating namespace I0429 08:50:35.623679 3791 namespace.go:98] [konflux-info] adding namespace I0429 08:50:35.624540 3791 namespace.go:102] [konflux-info] adding namespace took 839.361µs I0429 08:50:35.683922 3791 namespace.go:142] [konflux-info] updating namespace I0429 08:50:35.737592 3791 namespace.go:98] [namespace-lister] adding namespace I0429 08:50:35.738523 3791 namespace.go:102] [namespace-lister] adding namespace took 909.315µs I0429 08:50:35.781929 3791 namespace.go:98] [enterprise-contract-service] adding namespace I0429 08:50:35.782756 3791 namespace.go:102] [enterprise-contract-service] adding namespace took 807.652µs I0429 08:50:35.800488 3791 namespace.go:142] [konflux-info] updating namespace I0429 08:50:35.915742 3791 namespace.go:142] [image-controller] updating namespace I0429 08:50:35.967888 3791 namespace.go:142] [namespace-lister] updating namespace I0429 08:50:36.111658 3791 namespace.go:142] [namespace-lister] updating namespace I0429 08:50:36.289091 3791 namespace.go:142] [enterprise-contract-service] updating namespace I0429 08:50:36.431636 3791 namespace.go:142] [enterprise-contract-service] updating namespace I0429 08:50:36.438133 3791 namespace.go:142] [default-tenant] updating namespace I0429 08:50:36.796178 3791 namespace.go:98] [segment-bridge] adding namespace I0429 08:50:36.797089 3791 namespace.go:102] [segment-bridge] adding namespace took 889.098µs I0429 08:50:37.011009 3791 namespace.go:142] [segment-bridge] updating namespace I0429 08:50:37.020478 3791 namespace.go:98] [konflux-cli] adding namespace I0429 08:50:37.021336 3791 namespace.go:102] [konflux-cli] adding namespace took 839.59µs I0429 08:50:37.127274 3791 namespace.go:142] [segment-bridge] updating namespace I0429 08:50:37.241389 3791 namespace.go:142] [konflux-cli] updating namespace I0429 08:50:37.335014 3791 namespace.go:142] [konflux-cli] updating namespace I0429 08:50:37.413426 3791 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0429 08:50:37.414010 3791 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0429 08:50:37.414027 3791 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0429 08:50:37.416813 3791 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0429 08:50:37.472448 3791 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-696b855848-fqr9r] creating logical port namespace-lister_namespace-lister-696b855848-fqr9r for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:37.472561 3791 kube.go:256] Updating pod namespace-lister/namespace-lister-696b855848-fqr9r I0429 08:50:37.490840 3791 pod.go:62] [namespace-lister/namespace-lister-696b855848-fqr9r] pod update took 18.299483ms I0429 08:50:37.490866 3791 base_network_controller_pods.go:951] [default/namespace-lister/namespace-lister-696b855848-fqr9r] addLogicalPort annotation time took 18.344779ms I0429 08:50:37.492195 3791 pods.go:271] [namespace-lister/namespace-lister-696b855848-fqr9r] addLogicalPort took 19.765902ms, libovsdb time 787.088µs I0429 08:50:37.512434 3791 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0429 08:50:37.512459 3791 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0429 08:50:37.513793 3791 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0429 08:50:38.132972 3791 cni.go:452] [namespace-lister/namespace-lister-696b855848-fqr9r 0aed861a14d7ef5d2d3671af5347be6029124700d0240aee9772174f066315d9 network default NAD default NAD key ] ADD starting CNI request [namespace-lister/namespace-lister-696b855848-fqr9r 0aed861a14d7ef5d2d3671af5347be6029124700d0240aee9772174f066315d9 network default NAD default NAD key ] I0429 08:50:38.141871 3791 helper_linux.go:556] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-696b855848-fqr9r, hostIfaceName: 0aed861a14d7ef5, network: default, NAD default, SandboxID: "0aed861a14d7ef5d2d3671af5347be6029124700d0240aee9772174f066315d9", PCI device ID: , UID: "3e0a10e7-8913-44a7-9a39-4af3c41d1a06", MAC: 0a:58:0a:80:00:30, IPs: [10.128.0.48/23] I0429 08:50:38.198067 3791 cni.go:473] [namespace-lister/namespace-lister-696b855848-fqr9r 0aed861a14d7ef5d2d3671af5347be6029124700d0240aee9772174f066315d9 network default NAD default NAD key default] ADD finished CNI request [namespace-lister/namespace-lister-696b855848-fqr9r 0aed861a14d7ef5d2d3671af5347be6029124700d0240aee9772174f066315d9 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a6:8c:c9:5d:c1:49\",\"name\":\"0aed861a14d7ef5\"},{\"mac\":\"0a:58:0a:80:00:30\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f660bed5-840b-4140-a780-03d452b307a1\"}],\"ips\":[{\"address\":\"10.128.0.48/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:38.448624 3791 base_network_controller_pods.go:487] [default/image-controller/image-controller-controller-manager-6f859968fc-ndfpp] creating logical port image-controller_image-controller-controller-manager-6f859968fc-ndfpp for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:38.448747 3791 kube.go:256] Updating pod image-controller/image-controller-controller-manager-6f859968fc-ndfpp I0429 08:50:38.472796 3791 pod.go:62] [image-controller/image-controller-controller-manager-6f859968fc-ndfpp] pod update took 24.076341ms I0429 08:50:38.472818 3791 base_network_controller_pods.go:951] [default/image-controller/image-controller-controller-manager-6f859968fc-ndfpp] addLogicalPort annotation time took 24.102278ms I0429 08:50:38.474164 3791 pods.go:271] [image-controller/image-controller-controller-manager-6f859968fc-ndfpp] addLogicalPort took 25.555033ms, libovsdb time 931.183µs I0429 08:50:38.498893 3791 namespace.go:142] [build-service] updating namespace I0429 08:50:38.612891 3791 base_network_controller_pods.go:487] [default/konflux-ui/dex-7f7d44676c-bb9cj] creating logical port konflux-ui_dex-7f7d44676c-bb9cj for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:38.613002 3791 kube.go:256] Updating pod konflux-ui/dex-7f7d44676c-bb9cj I0429 08:50:38.630026 3791 pod.go:62] [konflux-ui/dex-7f7d44676c-bb9cj] pod update took 17.04669ms I0429 08:50:38.630050 3791 base_network_controller_pods.go:951] [default/konflux-ui/dex-7f7d44676c-bb9cj] addLogicalPort annotation time took 17.076839ms I0429 08:50:38.631315 3791 pods.go:271] [konflux-ui/dex-7f7d44676c-bb9cj] addLogicalPort took 18.435284ms, libovsdb time 872.769µs I0429 08:50:38.743138 3791 base_network_controller_pods.go:487] [default/konflux-ui/proxy-579fc6d5dd-s9hc4] creating logical port konflux-ui_proxy-579fc6d5dd-s9hc4 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:38.743235 3791 kube.go:256] Updating pod konflux-ui/proxy-579fc6d5dd-s9hc4 I0429 08:50:38.762292 3791 pod.go:62] [konflux-ui/proxy-579fc6d5dd-s9hc4] pod update took 19.090503ms I0429 08:50:38.762315 3791 base_network_controller_pods.go:951] [default/konflux-ui/proxy-579fc6d5dd-s9hc4] addLogicalPort annotation time took 19.117728ms I0429 08:50:38.763639 3791 pods.go:271] [konflux-ui/proxy-579fc6d5dd-s9hc4] addLogicalPort took 20.512833ms, libovsdb time 914.822µs I0429 08:50:38.809338 3791 cni.go:452] [image-controller/image-controller-controller-manager-6f859968fc-ndfpp 65aecf6da6e814ca584687abe03fb9202809fbc9be5aa07081a3ee2307501894 network default NAD default NAD key ] ADD starting CNI request [image-controller/image-controller-controller-manager-6f859968fc-ndfpp 65aecf6da6e814ca584687abe03fb9202809fbc9be5aa07081a3ee2307501894 network default NAD default NAD key ] I0429 08:50:38.820794 3791 helper_linux.go:556] ConfigureOVS: namespace: image-controller, podName: image-controller-controller-manager-6f859968fc-ndfpp, hostIfaceName: 65aecf6da6e814c, network: default, NAD default, SandboxID: "65aecf6da6e814ca584687abe03fb9202809fbc9be5aa07081a3ee2307501894", PCI device ID: , UID: "ba2efc66-5358-4d38-999d-eb423e312ba1", MAC: 0a:58:0a:80:00:31, IPs: [10.128.0.49/23] I0429 08:50:38.872296 3791 cni.go:473] [image-controller/image-controller-controller-manager-6f859968fc-ndfpp 65aecf6da6e814ca584687abe03fb9202809fbc9be5aa07081a3ee2307501894 network default NAD default NAD key default] ADD finished CNI request [image-controller/image-controller-controller-manager-6f859968fc-ndfpp 65aecf6da6e814ca584687abe03fb9202809fbc9be5aa07081a3ee2307501894 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:6c:f4:bc:bd:d4\",\"name\":\"65aecf6da6e814c\"},{\"mac\":\"0a:58:0a:80:00:31\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1cd61739-ed63-4de3-9b14-b5714928441d\"}],\"ips\":[{\"address\":\"10.128.0.49/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:40.133040 3791 cni.go:452] [tsf-tas/tuf-repository-init-2tl66-xvrfh 3e5b12765cf242ea4455f7a2790443371022684848a9dcd38fbab8ff20971320 network default NAD default NAD key ] DEL starting CNI request [tsf-tas/tuf-repository-init-2tl66-xvrfh 3e5b12765cf242ea4455f7a2790443371022684848a9dcd38fbab8ff20971320 network default NAD default NAD key ] I0429 08:50:40.140544 3791 namespace.go:142] [default-tenant] updating namespace W0429 08:50:40.164030 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0429 08:50:40.226236 3791 cni.go:473] [tsf-tas/tuf-repository-init-2tl66-xvrfh 3e5b12765cf242ea4455f7a2790443371022684848a9dcd38fbab8ff20971320 network default NAD default NAD key default] DEL finished CNI request [tsf-tas/tuf-repository-init-2tl66-xvrfh 3e5b12765cf242ea4455f7a2790443371022684848a9dcd38fbab8ff20971320 network default NAD default NAD key default], result "{}", err I0429 08:50:40.250296 3791 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4] creating logical port integration-service_integration-service-controller-manager-7b666bf6d8-f2zk4 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:40.250389 3791 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 I0429 08:50:40.264138 3791 pod.go:62] [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4] pod update took 13.772719ms I0429 08:50:40.264165 3791 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4] addLogicalPort annotation time took 13.804661ms I0429 08:50:40.265671 3791 pods.go:271] [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4] addLogicalPort took 15.387016ms, libovsdb time 932.912µs I0429 08:50:40.310269 3791 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-2tl66-xvrfh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:50:40.310305 3791 pods.go:176] Deleting pod: tsf-tas/tuf-repository-init-2tl66-xvrfh I0429 08:50:40.311725 3791 pods.go:236] Attempting to release IPs for pod: tsf-tas/tuf-repository-init-2tl66-xvrfh, ips: 10.128.0.47 I0429 08:50:40.311755 3791 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-2tl66-xvrfh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:50:40.893183 3791 namespace.go:142] [enterprise-contract-service] updating namespace I0429 08:50:41.009033 3791 base_network_controller_pods.go:487] [default/build-service/build-service-controller-manager-7cbf488fcf-5jkgn] creating logical port build-service_build-service-controller-manager-7cbf488fcf-5jkgn for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:41.009127 3791 kube.go:256] Updating pod build-service/build-service-controller-manager-7cbf488fcf-5jkgn I0429 08:50:41.031135 3791 pod.go:62] [build-service/build-service-controller-manager-7cbf488fcf-5jkgn] pod update took 22.035207ms I0429 08:50:41.031160 3791 base_network_controller_pods.go:951] [default/build-service/build-service-controller-manager-7cbf488fcf-5jkgn] addLogicalPort annotation time took 22.066829ms I0429 08:50:41.032498 3791 pods.go:271] [build-service/build-service-controller-manager-7cbf488fcf-5jkgn] addLogicalPort took 23.483088ms, libovsdb time 879.009µs W0429 08:50:41.153542 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0429 08:50:41.220513 3791 namespace.go:142] [image-controller] updating namespace I0429 08:50:41.345074 3791 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-2tl66-xvrfh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:50:41.345100 3791 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-2tl66-xvrfh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:50:41.421178 3791 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-6bfdb6bcb6-ztxj7] creating logical port release-service_release-service-controller-manager-6bfdb6bcb6-ztxj7 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:41.421270 3791 kube.go:256] Updating pod release-service/release-service-controller-manager-6bfdb6bcb6-ztxj7 I0429 08:50:41.440003 3791 pod.go:62] [release-service/release-service-controller-manager-6bfdb6bcb6-ztxj7] pod update took 18.756515ms I0429 08:50:41.440031 3791 base_network_controller_pods.go:951] [default/release-service/release-service-controller-manager-6bfdb6bcb6-ztxj7] addLogicalPort annotation time took 18.786505ms I0429 08:50:41.441356 3791 pods.go:271] [release-service/release-service-controller-manager-6bfdb6bcb6-ztxj7] addLogicalPort took 20.201224ms, libovsdb time 844.53µs I0429 08:50:41.550874 3791 namespace.go:142] [integration-service] updating namespace I0429 08:50:41.596459 3791 cni.go:452] [build-service/build-service-controller-manager-7cbf488fcf-5jkgn 8124a6edd6578ee14aa257a65da368cf2b05eca9da197ce175258febb328271e network default NAD default NAD key ] ADD starting CNI request [build-service/build-service-controller-manager-7cbf488fcf-5jkgn 8124a6edd6578ee14aa257a65da368cf2b05eca9da197ce175258febb328271e network default NAD default NAD key ] I0429 08:50:41.608536 3791 helper_linux.go:556] ConfigureOVS: namespace: build-service, podName: build-service-controller-manager-7cbf488fcf-5jkgn, hostIfaceName: 8124a6edd6578ee, network: default, NAD default, SandboxID: "8124a6edd6578ee14aa257a65da368cf2b05eca9da197ce175258febb328271e", PCI device ID: , UID: "e6f6f81d-9869-4fe6-bab5-61a9e3083fd1", MAC: 0a:58:0a:80:00:35, IPs: [10.128.0.53/23] I0429 08:50:41.662528 3791 cni.go:473] [build-service/build-service-controller-manager-7cbf488fcf-5jkgn 8124a6edd6578ee14aa257a65da368cf2b05eca9da197ce175258febb328271e network default NAD default NAD key default] ADD finished CNI request [build-service/build-service-controller-manager-7cbf488fcf-5jkgn 8124a6edd6578ee14aa257a65da368cf2b05eca9da197ce175258febb328271e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:78:70:00:f9:de\",\"name\":\"8124a6edd6578ee\"},{\"mac\":\"0a:58:0a:80:00:35\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d1634e5b-66fe-419c-97df-f0d1bf126dba\"}],\"ips\":[{\"address\":\"10.128.0.53/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:41.810354 3791 namespace.go:142] [konflux-cli] updating namespace I0429 08:50:42.073610 3791 namespace.go:142] [konflux-info] updating namespace W0429 08:50:42.144821 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service I0429 08:50:42.251522 3791 namespace.go:142] [namespace-lister] updating namespace I0429 08:50:42.388096 3791 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-7b99557495-7pbm2] creating logical port integration-service_integration-service-controller-manager-7b99557495-7pbm2 for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:42.388229 3791 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7b99557495-7pbm2 I0429 08:50:42.403545 3791 pod.go:62] [integration-service/integration-service-controller-manager-7b99557495-7pbm2] pod update took 15.338295ms I0429 08:50:42.403571 3791 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-7b99557495-7pbm2] addLogicalPort annotation time took 15.369058ms I0429 08:50:42.405085 3791 pods.go:271] [integration-service/integration-service-controller-manager-7b99557495-7pbm2] addLogicalPort took 17.009754ms, libovsdb time 1.015663ms I0429 08:50:42.443654 3791 namespace.go:142] [release-service] updating namespace I0429 08:50:42.655529 3791 namespace.go:142] [segment-bridge] updating namespace I0429 08:50:42.779212 3791 namespace.go:142] [build-service] updating namespace I0429 08:50:42.804700 3791 namespace.go:142] [default-tenant] updating namespace I0429 08:50:42.828795 3791 namespace.go:142] [enterprise-contract-service] updating namespace I0429 08:50:42.840908 3791 namespace.go:142] [image-controller] updating namespace I0429 08:50:42.855802 3791 namespace.go:142] [integration-service] updating namespace I0429 08:50:42.887407 3791 namespace.go:142] [konflux-cli] updating namespace I0429 08:50:42.926932 3791 namespace.go:142] [konflux-info] updating namespace I0429 08:50:42.992875 3791 namespace.go:142] [namespace-lister] updating namespace I0429 08:50:43.023876 3791 namespace.go:142] [release-service] updating namespace I0429 08:50:43.063411 3791 namespace.go:142] [segment-bridge] updating namespace I0429 08:50:43.175334 3791 namespace.go:142] [build-service] updating namespace I0429 08:50:43.245001 3791 namespace.go:142] [default-tenant] updating namespace I0429 08:50:43.330418 3791 namespace.go:142] [enterprise-contract-service] updating namespace I0429 08:50:43.408582 3791 namespace.go:142] [image-controller] updating namespace I0429 08:50:43.473979 3791 namespace.go:142] [integration-service] updating namespace I0429 08:50:43.543784 3791 namespace.go:142] [konflux-cli] updating namespace I0429 08:50:43.601636 3791 namespace.go:142] [konflux-info] updating namespace I0429 08:50:43.701118 3791 namespace.go:142] [namespace-lister] updating namespace I0429 08:50:43.787158 3791 namespace.go:142] [release-service] updating namespace I0429 08:50:43.898210 3791 namespace.go:142] [segment-bridge] updating namespace I0429 08:50:44.510167 3791 base_network_controller_pods.go:487] [default/tsf-tas/tuf-97fc8ff76-qq7hk] creating logical port tsf-tas_tuf-97fc8ff76-qq7hk for pod on switch ip-10-0-0-38.ec2.internal I0429 08:50:44.510291 3791 kube.go:256] Updating pod tsf-tas/tuf-97fc8ff76-qq7hk I0429 08:50:44.526265 3791 pod.go:62] [tsf-tas/tuf-97fc8ff76-qq7hk] pod update took 16.030209ms I0429 08:50:44.526292 3791 base_network_controller_pods.go:951] [default/tsf-tas/tuf-97fc8ff76-qq7hk] addLogicalPort annotation time took 16.060737ms I0429 08:50:44.527946 3791 pods.go:271] [tsf-tas/tuf-97fc8ff76-qq7hk] addLogicalPort took 17.795992ms, libovsdb time 1.02726ms W0429 08:50:45.165961 3791 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0429 08:50:46.352433 3791 cni.go:452] [integration-service/integration-service-controller-manager-7b99557495-7pbm2 331caa985e5aaf7d3343a0490dc954a37708f55033d472856ca793ebce47e810 network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-7b99557495-7pbm2 331caa985e5aaf7d3343a0490dc954a37708f55033d472856ca793ebce47e810 network default NAD default NAD key ] I0429 08:50:46.361554 3791 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-7b99557495-7pbm2, hostIfaceName: 331caa985e5aaf7, network: default, NAD default, SandboxID: "331caa985e5aaf7d3343a0490dc954a37708f55033d472856ca793ebce47e810", PCI device ID: , UID: "0d9b10ff-4548-49ca-a5f0-620816c7f19d", MAC: 0a:58:0a:80:00:37, IPs: [10.128.0.55/23] I0429 08:50:46.413615 3791 cni.go:473] [integration-service/integration-service-controller-manager-7b99557495-7pbm2 331caa985e5aaf7d3343a0490dc954a37708f55033d472856ca793ebce47e810 network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-7b99557495-7pbm2 331caa985e5aaf7d3343a0490dc954a37708f55033d472856ca793ebce47e810 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"42:de:0d:a8:14:30\",\"name\":\"331caa985e5aaf7\"},{\"mac\":\"0a:58:0a:80:00:37\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/aa3ca1ee-1f82-4c86-ae65-9dab75cc15df\"}],\"ips\":[{\"address\":\"10.128.0.55/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:46.608428 3791 cni.go:452] [konflux-ui/proxy-579fc6d5dd-s9hc4 c2367effb4057b0e7c2fad260f2ffdfdbff69e3389677f1f04125c0635817201 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-579fc6d5dd-s9hc4 c2367effb4057b0e7c2fad260f2ffdfdbff69e3389677f1f04125c0635817201 network default NAD default NAD key ] I0429 08:50:46.618165 3791 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-579fc6d5dd-s9hc4, hostIfaceName: c2367effb4057b0, network: default, NAD default, SandboxID: "c2367effb4057b0e7c2fad260f2ffdfdbff69e3389677f1f04125c0635817201", PCI device ID: , UID: "74d589d6-a58e-4ce5-9987-8790baec423c", MAC: 0a:58:0a:80:00:33, IPs: [10.128.0.51/23] I0429 08:50:46.668725 3791 cni.go:473] [konflux-ui/proxy-579fc6d5dd-s9hc4 c2367effb4057b0e7c2fad260f2ffdfdbff69e3389677f1f04125c0635817201 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-579fc6d5dd-s9hc4 c2367effb4057b0e7c2fad260f2ffdfdbff69e3389677f1f04125c0635817201 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:c2:61:22:ca:ee\",\"name\":\"c2367effb4057b0\"},{\"mac\":\"0a:58:0a:80:00:33\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/eef56b92-1853-4eb8-a6c0-2b7d98b9d59a\"}],\"ips\":[{\"address\":\"10.128.0.51/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:46.774429 3791 cni.go:452] [konflux-ui/dex-7f7d44676c-bb9cj 8c5e53ca983d12948c7dcf063450fac03594dc289ffaac3859f3781b40ade1db network default NAD default NAD key ] ADD starting CNI request [konflux-ui/dex-7f7d44676c-bb9cj 8c5e53ca983d12948c7dcf063450fac03594dc289ffaac3859f3781b40ade1db network default NAD default NAD key ] I0429 08:50:46.783764 3791 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: dex-7f7d44676c-bb9cj, hostIfaceName: 8c5e53ca983d129, network: default, NAD default, SandboxID: "8c5e53ca983d12948c7dcf063450fac03594dc289ffaac3859f3781b40ade1db", PCI device ID: , UID: "d8ef01b6-a9b6-4c13-9f12-aeadcc413d39", MAC: 0a:58:0a:80:00:32, IPs: [10.128.0.50/23] I0429 08:50:46.833861 3791 cni.go:473] [konflux-ui/dex-7f7d44676c-bb9cj 8c5e53ca983d12948c7dcf063450fac03594dc289ffaac3859f3781b40ade1db network default NAD default NAD key default] ADD finished CNI request [konflux-ui/dex-7f7d44676c-bb9cj 8c5e53ca983d12948c7dcf063450fac03594dc289ffaac3859f3781b40ade1db network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:cc:40:a7:1f:09\",\"name\":\"8c5e53ca983d129\"},{\"mac\":\"0a:58:0a:80:00:32\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dcc1fec2-4163-49c2-843d-c424e0b75705\"}],\"ips\":[{\"address\":\"10.128.0.50/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:48.240574 3791 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 db2c74e98fab61a96446f72cc5f9f5df603ba91eae516103a0bbb4f5cbec031b network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 db2c74e98fab61a96446f72cc5f9f5df603ba91eae516103a0bbb4f5cbec031b network default NAD default NAD key ] I0429 08:50:48.251336 3791 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-7b666bf6d8-f2zk4, hostIfaceName: db2c74e98fab61a, network: default, NAD default, SandboxID: "db2c74e98fab61a96446f72cc5f9f5df603ba91eae516103a0bbb4f5cbec031b", PCI device ID: , UID: "f24247d1-4026-4ab4-8080-e256b83eb8f0", MAC: 0a:58:0a:80:00:34, IPs: [10.128.0.52/23] I0429 08:50:48.316481 3791 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 db2c74e98fab61a96446f72cc5f9f5df603ba91eae516103a0bbb4f5cbec031b network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 db2c74e98fab61a96446f72cc5f9f5df603ba91eae516103a0bbb4f5cbec031b network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:27:a4:dd:30:3c\",\"name\":\"db2c74e98fab61a\"},{\"mac\":\"0a:58:0a:80:00:34\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/4b04259a-9b46-4a24-8446-51d40eee1c6c\"}],\"ips\":[{\"address\":\"10.128.0.52/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:50:49.192190 3791 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0429 08:50:51.207339 3791 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0429 08:50:51.207355 3791 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy I0429 08:50:51.989087 3791 cni.go:452] [release-service/release-service-controller-manager-6bfdb6bcb6-ztxj7 aa7a83a071930dc51de9114327dd327f356109c7f9b3ccb0b6cc43c8d989798d network default NAD default NAD key ] ADD starting CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-ztxj7 aa7a83a071930dc51de9114327dd327f356109c7f9b3ccb0b6cc43c8d989798d network default NAD default NAD key ] I0429 08:50:51.998762 3791 helper_linux.go:556] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-6bfdb6bcb6-ztxj7, hostIfaceName: aa7a83a071930dc, network: default, NAD default, SandboxID: "aa7a83a071930dc51de9114327dd327f356109c7f9b3ccb0b6cc43c8d989798d", PCI device ID: , UID: "e5c35add-d98b-40e1-a17d-46b389344a59", MAC: 0a:58:0a:80:00:36, IPs: [10.128.0.54/23] I0429 08:50:52.056538 3791 cni.go:473] [release-service/release-service-controller-manager-6bfdb6bcb6-ztxj7 aa7a83a071930dc51de9114327dd327f356109c7f9b3ccb0b6cc43c8d989798d network default NAD default NAD key default] ADD finished CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-ztxj7 aa7a83a071930dc51de9114327dd327f356109c7f9b3ccb0b6cc43c8d989798d network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:85:29:14:dd:89\",\"name\":\"aa7a83a071930dc\"},{\"mac\":\"0a:58:0a:80:00:36\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/81bdab12-b211-4b07-ac77-9804844314cd\"}],\"ips\":[{\"address\":\"10.128.0.54/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 08:50:52.673011 3791 cni.go:452] [tsf-tas/tuf-97fc8ff76-qq7hk 026feedca0594d8b5e37a0ff4537456a6802dda82cd6039f1abf68ab69e504f0 network default NAD default NAD key ] ADD starting CNI request [tsf-tas/tuf-97fc8ff76-qq7hk 026feedca0594d8b5e37a0ff4537456a6802dda82cd6039f1abf68ab69e504f0 network default NAD default NAD key ] I0429 08:50:52.682704 3791 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: tuf-97fc8ff76-qq7hk, hostIfaceName: 026feedca0594d8, network: default, NAD default, SandboxID: "026feedca0594d8b5e37a0ff4537456a6802dda82cd6039f1abf68ab69e504f0", PCI device ID: , UID: "c1d38627-d7e1-4b4c-938d-801950caa576", MAC: 0a:58:0a:80:00:38, IPs: [10.128.0.56/23] I0429 08:50:52.736390 3791 cni.go:473] [tsf-tas/tuf-97fc8ff76-qq7hk 026feedca0594d8b5e37a0ff4537456a6802dda82cd6039f1abf68ab69e504f0 network default NAD default NAD key default] ADD finished CNI request [tsf-tas/tuf-97fc8ff76-qq7hk 026feedca0594d8b5e37a0ff4537456a6802dda82cd6039f1abf68ab69e504f0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:a7:be:26:77:0c\",\"name\":\"026feedca0594d8\"},{\"mac\":\"0a:58:0a:80:00:38\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2928aae1-b0a0-44b7-951b-881a64a0d822\"}],\"ips\":[{\"address\":\"10.128.0.56/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err W0429 08:50:53.154318 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0429 08:50:53.231151 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0429 08:50:53.231232 3791 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0429 08:50:53.255567 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0429 08:50:53.257654 3791 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0429 08:50:53.313038 3791 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0429 08:50:53.313053 3791 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0429 08:50:55.247366 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0429 08:50:55.273242 3791 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0429 08:50:55.275069 3791 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0429 08:50:56.168789 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8443: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service I0429 08:50:57.860596 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=58 W0429 08:51:04.262204 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/dex W0429 08:51:04.292806 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0429 08:51:04.304494 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0429 08:51:05.052721 3791 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 db2c74e98fab61a96446f72cc5f9f5df603ba91eae516103a0bbb4f5cbec031b network default NAD default NAD key ] DEL starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 db2c74e98fab61a96446f72cc5f9f5df603ba91eae516103a0bbb4f5cbec031b network default NAD default NAD key ] I0429 08:51:05.129666 3791 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 db2c74e98fab61a96446f72cc5f9f5df603ba91eae516103a0bbb4f5cbec031b network default NAD default NAD key default] DEL finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 db2c74e98fab61a96446f72cc5f9f5df603ba91eae516103a0bbb4f5cbec031b network default NAD default NAD key default], result "{}", err I0429 08:51:05.209222 3791 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:51:05.209258 3791 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 I0429 08:51:05.210595 3791 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4, ips: 10.128.0.52 I0429 08:51:05.210618 3791 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:51:05.320525 3791 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:51:05.320552 3791 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:51:05.327875 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 I0429 08:51:05.327903 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-f2zk4 W0429 08:51:06.263039 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0429 08:51:06.266811 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0429 08:51:06.286955 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service tsf-tas/tuf I0429 08:51:23.972777 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=8 I0429 08:51:30.963538 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=9 W0429 08:51:32.288277 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0429 08:51:32.288294 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0429 08:51:32.288304 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0429 08:51:32.288308 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0429 08:51:32.311084 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0429 08:51:32.311100 3791 util.go:959] Failed to build node endpoints for node ip-10-0-0-38.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0429 08:52:02.219154 3791 cni.go:452] [konflux-ui/proxy-579fc6d5dd-s9hc4 c2367effb4057b0e7c2fad260f2ffdfdbff69e3389677f1f04125c0635817201 network default NAD default NAD key ] DEL starting CNI request [konflux-ui/proxy-579fc6d5dd-s9hc4 c2367effb4057b0e7c2fad260f2ffdfdbff69e3389677f1f04125c0635817201 network default NAD default NAD key ] I0429 08:52:02.319395 3791 cni.go:473] [konflux-ui/proxy-579fc6d5dd-s9hc4 c2367effb4057b0e7c2fad260f2ffdfdbff69e3389677f1f04125c0635817201 network default NAD default NAD key default] DEL finished CNI request [konflux-ui/proxy-579fc6d5dd-s9hc4 c2367effb4057b0e7c2fad260f2ffdfdbff69e3389677f1f04125c0635817201 network default NAD default NAD key default], result "{}", err I0429 08:52:02.382079 3791 obj_retry.go:498] Detected object konflux-ui/proxy-579fc6d5dd-s9hc4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:52:02.382131 3791 pods.go:176] Deleting pod: konflux-ui/proxy-579fc6d5dd-s9hc4 I0429 08:52:02.383788 3791 pods.go:236] Attempting to release IPs for pod: konflux-ui/proxy-579fc6d5dd-s9hc4, ips: 10.128.0.51 I0429 08:52:02.383816 3791 obj_retry.go:498] Detected object konflux-ui/proxy-579fc6d5dd-s9hc4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:52:02.645486 3791 obj_retry.go:492] Detected object konflux-ui/proxy-579fc6d5dd-s9hc4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:52:02.645510 3791 obj_retry.go:492] Detected object konflux-ui/proxy-579fc6d5dd-s9hc4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:52:02.655491 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-579fc6d5dd-s9hc4 I0429 08:52:02.655519 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-579fc6d5dd-s9hc4 I0429 08:52:07.764610 3791 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 08:52:07.765191 3791 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 08:52:07.785207 3791 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:52:07.785227 3791 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:52:07.792677 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0429 08:52:07.792706 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0429 08:52:37.796856 3791 namespace.go:98] [managed-tenant-tqyug] adding namespace I0429 08:52:37.797762 3791 namespace.go:102] [managed-tenant-tqyug] adding namespace took 878.109µs I0429 08:52:37.825384 3791 namespace.go:142] [managed-tenant-tqyug] updating namespace I0429 08:52:37.844603 3791 namespace.go:142] [managed-tenant-tqyug] updating namespace I0429 08:52:38.944063 3791 namespace.go:142] [managed-tenant-tqyug] updating namespace I0429 08:52:39.075185 3791 namespace.go:142] [managed-tenant-tqyug] updating namespace I0429 08:52:39.178167 3791 namespace.go:142] [managed-tenant-tqyug] updating namespace I0429 08:52:53.997958 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=8 I0429 08:53:15.954177 3791 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=5 I0429 08:54:03.990517 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 I0429 08:54:49.978647 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=10 I0429 08:54:50.860167 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=148 I0429 08:54:59.968767 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0429 08:55:01.868993 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=405 I0429 08:55:11.956164 3791 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0429 08:55:15.862892 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=81 I0429 08:55:48.984688 3791 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=10 I0429 08:56:00.860064 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=9 I0429 08:56:17.862852 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=55 I0429 08:56:42.976239 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=6 I0429 08:57:07.959446 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=11 I0429 08:59:04.000468 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=7 W0429 08:59:15.522206 3791 duration.go:461] Config duration recorder: measurement expired for pod/openshift-image-registry/image-registry-c64d887fc-nvgqf I0429 08:59:22.965994 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=8 I0429 08:59:33.862933 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=40 I0429 08:59:48.047906 3791 obj_retry.go:492] Detected object tsf-tpa/create-db-js87p of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:59:48.047949 3791 obj_retry.go:492] Detected object tsf-tpa/create-db-js87p of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:59:48.052402 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-db-js87p I0429 08:59:48.052416 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-db-js87p I0429 08:59:51.028446 3791 obj_retry.go:492] Detected object tsf-tpa/migrate-db-tdxfr of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:59:51.028497 3791 obj_retry.go:492] Detected object tsf-tpa/migrate-db-tdxfr of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:59:51.032652 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/migrate-db-tdxfr I0429 08:59:51.032671 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/migrate-db-tdxfr I0429 08:59:53.980868 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0429 08:59:57.034775 3791 obj_retry.go:492] Detected object tsf-tpa/create-importers-j4z9z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:59:57.034848 3791 obj_retry.go:492] Detected object tsf-tpa/create-importers-j4z9z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 08:59:57.040150 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/create-importers-j4z9z I0429 08:59:57.040171 3791 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/create-importers-j4z9z I0429 09:00:00.186750 3791 base_network_controller_pods.go:487] [default/segment-bridge/segment-bridge-29624220-wbft8] creating logical port segment-bridge_segment-bridge-29624220-wbft8 for pod on switch ip-10-0-0-38.ec2.internal I0429 09:00:00.186891 3791 kube.go:256] Updating pod segment-bridge/segment-bridge-29624220-wbft8 I0429 09:00:00.225744 3791 base_network_controller_pods.go:487] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf] creating logical port openshift-backplane_osd-delete-backplane-serviceaccounts-29624220-ct9bf for pod on switch ip-10-0-0-38.ec2.internal I0429 09:00:00.225873 3791 kube.go:256] Updating pod openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf I0429 09:00:00.228181 3791 pod.go:62] [segment-bridge/segment-bridge-29624220-wbft8] pod update took 41.319661ms I0429 09:00:00.228201 3791 base_network_controller_pods.go:951] [default/segment-bridge/segment-bridge-29624220-wbft8] addLogicalPort annotation time took 41.343929ms I0429 09:00:00.229644 3791 pods.go:271] [segment-bridge/segment-bridge-29624220-wbft8] addLogicalPort took 42.90961ms, libovsdb time 901.372µs I0429 09:00:00.249199 3791 pod.go:62] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf] pod update took 23.348299ms I0429 09:00:00.249223 3791 base_network_controller_pods.go:951] [default/openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf] addLogicalPort annotation time took 23.393675ms I0429 09:00:00.250650 3791 pods.go:271] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf] addLogicalPort took 24.919842ms, libovsdb time 936.967µs I0429 09:00:00.548768 3791 cni.go:452] [segment-bridge/segment-bridge-29624220-wbft8 8b22f45e1fc369847e49e3994de1e0e6bd3d4b1fb221d6d678bde7d8a63c3dc4 network default NAD default NAD key ] ADD starting CNI request [segment-bridge/segment-bridge-29624220-wbft8 8b22f45e1fc369847e49e3994de1e0e6bd3d4b1fb221d6d678bde7d8a63c3dc4 network default NAD default NAD key ] I0429 09:00:00.558806 3791 helper_linux.go:556] ConfigureOVS: namespace: segment-bridge, podName: segment-bridge-29624220-wbft8, hostIfaceName: 8b22f45e1fc3698, network: default, NAD default, SandboxID: "8b22f45e1fc369847e49e3994de1e0e6bd3d4b1fb221d6d678bde7d8a63c3dc4", PCI device ID: , UID: "a08ae810-7ef8-49f0-942c-1b37d97172d3", MAC: 0a:58:0a:80:00:39, IPs: [10.128.0.57/23] I0429 09:00:00.598069 3791 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf 8df72ea47e3dd0b07bdfc93324f85d31587709f7d62a4a52556d905464691463 network default NAD default NAD key ] ADD starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf 8df72ea47e3dd0b07bdfc93324f85d31587709f7d62a4a52556d905464691463 network default NAD default NAD key ] I0429 09:00:00.608793 3791 helper_linux.go:556] ConfigureOVS: namespace: openshift-backplane, podName: osd-delete-backplane-serviceaccounts-29624220-ct9bf, hostIfaceName: 8df72ea47e3dd0b, network: default, NAD default, SandboxID: "8df72ea47e3dd0b07bdfc93324f85d31587709f7d62a4a52556d905464691463", PCI device ID: , UID: "ae0c46af-5aeb-4a25-abb6-e39acd6efd2b", MAC: 0a:58:0a:80:00:3a, IPs: [10.128.0.58/23] I0429 09:00:00.620507 3791 cni.go:473] [segment-bridge/segment-bridge-29624220-wbft8 8b22f45e1fc369847e49e3994de1e0e6bd3d4b1fb221d6d678bde7d8a63c3dc4 network default NAD default NAD key default] ADD finished CNI request [segment-bridge/segment-bridge-29624220-wbft8 8b22f45e1fc369847e49e3994de1e0e6bd3d4b1fb221d6d678bde7d8a63c3dc4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:77:b9:6f:fe:8a\",\"name\":\"8b22f45e1fc3698\"},{\"mac\":\"0a:58:0a:80:00:39\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b36d4ee4-a9cc-45f7-aeae-27809a2d9aef\"}],\"ips\":[{\"address\":\"10.128.0.57/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 09:00:00.870139 3791 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf 8df72ea47e3dd0b07bdfc93324f85d31587709f7d62a4a52556d905464691463 network default NAD default NAD key default] ADD finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf 8df72ea47e3dd0b07bdfc93324f85d31587709f7d62a4a52556d905464691463 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:c6:c0:ef:71:90\",\"name\":\"8df72ea47e3dd0b\"},{\"mac\":\"0a:58:0a:80:00:3a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/1dc19c52-6291-45e6-a772-537d912c36fb\"}],\"ips\":[{\"address\":\"10.128.0.58/23\",\"gateway\":\"10.128.0.1\",\"interface\":1}]}", err I0429 09:00:04.254946 3791 cni.go:452] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf 8df72ea47e3dd0b07bdfc93324f85d31587709f7d62a4a52556d905464691463 network default NAD default NAD key ] DEL starting CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf 8df72ea47e3dd0b07bdfc93324f85d31587709f7d62a4a52556d905464691463 network default NAD default NAD key ] I0429 09:00:04.343418 3791 cni.go:473] [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf 8df72ea47e3dd0b07bdfc93324f85d31587709f7d62a4a52556d905464691463 network default NAD default NAD key default] DEL finished CNI request [openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf 8df72ea47e3dd0b07bdfc93324f85d31587709f7d62a4a52556d905464691463 network default NAD default NAD key default], result "{}", err I0429 09:00:04.396751 3791 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 09:00:04.396814 3791 pods.go:176] Deleting pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf I0429 09:00:04.398165 3791 pods.go:236] Attempting to release IPs for pod: openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf, ips: 10.128.0.58 I0429 09:00:04.398197 3791 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 09:00:05.267710 3791 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 09:00:05.267766 3791 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624220-ct9bf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 09:00:42.971366 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0429 09:00:52.863331 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=6 I0429 09:01:18.987267 3791 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0429 09:02:42.871772 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=36 I0429 09:02:44.993050 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=9 I0429 09:03:08.961647 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=6 I0429 09:03:10.956490 3791 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=10 I0429 09:03:23.958469 3791 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9 I0429 09:03:35.866305 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=8 I0429 09:04:20.002593 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=6 I0429 09:04:33.866073 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=29 I0429 09:04:44.865734 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=10 I0429 09:05:22.968223 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=7 I0429 09:05:41.862612 3791 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=11 I0429 09:06:01.978580 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=10 I0429 09:06:27.973892 3791 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=6 I0429 09:06:56.989755 3791 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6