+ . /ovnkube-lib/ovnkube-lib.sh ++ set -x ++ K8S_NODE=ip-10-0-2-36.ec2.internal ++ [[ -n ip-10-0-2-36.ec2.internal ]] ++ [[ -f /env/ip-10-0-2-36.ec2.internal ]] ++ northd_pidfile=/var/run/ovn/ovn-northd.pid ++ controller_pidfile=/var/run/ovn/ovn-controller.pid ++ controller_logfile=/var/log/ovn/acl-audit-log.log ++ vswitch_dbsock=/var/run/openvswitch/db.sock ++ nbdb_pidfile=/var/run/ovn/ovnnb_db.pid ++ nbdb_sock=/var/run/ovn/ovnnb_db.sock ++ nbdb_ctl=/var/run/ovn/ovnnb_db.ctl ++ sbdb_pidfile=/var/run/ovn/ovnsb_db.pid ++ sbdb_sock=/var/run/ovn/ovnsb_db.sock ++ sbdb_ctl=/var/run/ovn/ovnsb_db.ctl + start-ovnkube-node 4 29103 29105 + local log_level=4 + local metrics_port=29103 + local ovn_metrics_port=29105 + ovn_advertised_udn_isolation_mode_flag= + openflow_probe_flag= + [[ 3 -ne 3 ]] + ovs-vsctl br-exists br-ex + add_garp_drop_flow br-ex + local bridge=br-ex + local cookie=0x0305 + local priority=499 ++ ovs-vsctl list-ports br-ex + for port_name in $(ovs-vsctl list-ports "$bridge") + [[ ens5 == *to-br-int ]] + cni-bin-copy + . /host/etc/os-release ++ NAME='Red Hat Enterprise Linux CoreOS' ++ VERSION='9.6.20260401-0 (Plow)' ++ ID=rhel ++ ID_LIKE=fedora ++ VERSION_ID=9.6 ++ PLATFORM_ID=platform:el9 ++ PRETTY_NAME='Red Hat Enterprise Linux CoreOS 9.6.20260401-0 (Plow)' ++ ANSI_COLOR='0;31' ++ LOGO=fedora-logo-icon ++ CPE_NAME=cpe:/o:redhat:enterprise_linux:9::baseos ++ HOME_URL=https://www.redhat.com/ ++ DOCUMENTATION_URL=https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/9 ++ BUG_REPORT_URL=https://issues.redhat.com/ ++ REDHAT_BUGZILLA_PRODUCT='Red Hat Enterprise Linux 9' ++ REDHAT_BUGZILLA_PRODUCT_VERSION=9.6 ++ REDHAT_SUPPORT_PRODUCT='Red Hat Enterprise Linux' ++ REDHAT_SUPPORT_PRODUCT_VERSION=9.6 ++ OSTREE_VERSION=9.6.20260401-0 ++ VARIANT=CoreOS ++ VARIANT_ID=coreos ++ OPENSHIFT_VERSION=4.21 + rhelmajor= + case "${ID}" in ++ echo 9.6 ++ cut -f 1 -d . + rhelmajor=9 + sourcedir=/usr/libexec/cni/ + case "${rhelmajor}" in + sourcedir=/usr/libexec/cni/rhel9 + cp -f /usr/libexec/cni/rhel9/ovn-k8s-cni-overlay /cni-bin-dir/ ++ date '+%m%d %H:%M:%S.%N' + echo 'I0429 06:48:16.837894194 - disable conntrack on geneve port' I0429 06:48:16.837894194 - disable conntrack on geneve port + iptables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + iptables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A PREROUTING -p udp --dport 6081 -j NOTRACK + ip6tables -t raw -A OUTPUT -p udp --dport 6081 -j NOTRACK ++ date '+%m%d %H:%M:%S.%N' I0429 06:48:16.901743752 - starting ovnkube-node + echo 'I0429 06:48:16.901743752 - starting ovnkube-node' + egress_features_enable_flag='--enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true' + init_ovnkube_controller='--init-ovnkube-controller ip-10-0-2-36.ec2.internal' + multi_external_gateway_enable_flag=--enable-multi-external-gateway=true + gateway_interface=br-ex + enable_multicast_flag=--enable-multicast + OVN_NODE_MODE=full + '[' full == dpu-host ']' + '[' shared == shared ']' + gateway_mode_flags='--gateway-mode shared --gateway-interface br-ex' + export_network_flows_flags= + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + [[ -n '' ]] + gw_interface_flag= + '[' -d /sys/class/net/br-ex1 ']' + node_mgmt_port_netdev_flags= + [[ -n '' ]] + [[ -n '' ]] + multi_network_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + multi_network_enabled_flag=--enable-multi-network + network_segmentation_enabled_flag=--enable-network-segmentation + route_advertisements_enable_flag= + [[ false == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag= + [[ true == \t\r\u\e ]] + preconfigured_udn_addresses_enable_flag=--enable-preconfigured-udn-addresses + network_observability_enabled_flag= + [[ false == \t\r\u\e ]] + multi_network_policy_enabled_flag= + [[ false == \t\r\u\e ]] + admin_network_policy_enabled_flag= + [[ true == \t\r\u\e ]] + [[ full != \d\p\u\-\h\o\s\t ]] + admin_network_policy_enabled_flag=--enable-admin-network-policy + dns_name_resolver_enabled_flag= + [[ false == \t\r\u\e ]] + ip_forwarding_flag= + '[' '' == Global ']' + ip_forwarding_flag=--disable-forwarding + sysctl -w net.ipv4.ip_forward=0 net.ipv4.ip_forward = 0 + sysctl -w net.ipv6.conf.all.forwarding=0 net.ipv6.conf.all.forwarding = 0 + [[ '' != '' ]] + [[ '' != '' ]] + NETWORK_NODE_IDENTITY_ENABLE= + [[ true == \t\r\u\e ]] + NETWORK_NODE_IDENTITY_ENABLE=' --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h ' + ovn_v4_join_subnet_opt= + [[ '' != '' ]] + ovn_v6_join_subnet_opt= + [[ '' != '' ]] + ovn_v4_masquerade_subnet_opt= + [[ 169.254.0.0/17 != '' ]] + ovn_v4_masquerade_subnet_opt='--gateway-v4-masquerade-subnet 169.254.0.0/17' + ovn_v6_masquerade_subnet_opt= + [[ fd69::/112 != '' ]] + ovn_v6_masquerade_subnet_opt='--gateway-v6-masquerade-subnet fd69::/112' + ovn_v4_transit_switch_subnet_opt= + [[ '' != '' ]] + ovn_v6_transit_switch_subnet_opt= + [[ '' != '' ]] + exec /usr/bin/ovnkube --init-ovnkube-controller ip-10-0-2-36.ec2.internal --init-node ip-10-0-2-36.ec2.internal --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-network-segmentation --enable-preconfigured-udn-addresses --enable-admin-network-policy --enable-multicast --zone ip-10-0-2-36.ec2.internal --enable-interconnect --acl-logging-rate-limit 20 --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h --gateway-v4-masquerade-subnet 169.254.0.0/17 --gateway-v6-masquerade-subnet fd69::/112 --enable-egress-ip=true --enable-egress-firewall=true --enable-egress-qos=true --enable-egress-service=true --enable-multi-external-gateway=true I0429 06:48:16.956312 3947 config.go:2726] Parsed config file /run/ovnkube-config/ovnkube.conf I0429 06:48:16.956383 3947 config.go:2727] Parsed config: {Default:{MTU:8901 RoutableMTU:0 ConntrackZone:64000 HostMasqConntrackZone:0 OVNMasqConntrackZone:0 HostNodePortConntrackZone:0 ReassemblyConntrackZone:0 EncapType:geneve EncapIP: EffectiveEncapIP: EncapPort:6081 InactivityProbe:100000 OpenFlowProbe:0 OfctrlWaitBeforeClear:0 MonitorAll:true OVSDBTxnTimeout:1m40s LFlowCacheEnable:true LFlowCacheLimit:0 LFlowCacheLimitKb:1048576 RawClusterSubnets:10.128.0.0/14/23 ClusterSubnets:[] EnableUDPAggregation:true Zone:global RawUDNAllowedDefaultServices:default/kubernetes,openshift-dns/dns-default UDNAllowedDefaultServices:[] Transport:geneve} Logging:{File: CNIFile: LibovsdbFile: Level:4 LogFileMaxSize:100 LogFileMaxBackups:5 LogFileMaxAge:5 ACLLoggingRateLimit:20} Monitoring:{RawNetFlowTargets: RawSFlowTargets: RawIPFIXTargets: NetFlowTargets:[] SFlowTargets:[] IPFIXTargets:[]} IPFIX:{Sampling:400 CacheActiveTimeout:60 CacheMaxFlows:0} CNI:{ConfDir:/etc/cni/net.d Plugin:ovn-k8s-cni-overlay} OVNKubernetesFeature:{EnableAdminNetworkPolicy:false EnableEgressIP:false EgressIPReachabiltyTotalTimeout:1 EnableEgressFirewall:false EnableEgressQoS:false EnableEgressService:false EgressIPNodeHealthCheckPort:9107 EnableMultiNetwork:false EnableNetworkSegmentation:true EnableNetworkConnect:false EnablePreconfiguredUDNAddresses:true EnableRouteAdvertisements:false EnableEVPN:false EnableMultiNetworkPolicy:false EnableStatelessNetPol:false EnableInterconnect:false EnableMultiExternalGateway:false EnablePersistentIPs:false EnableDNSNameResolver:false EnableServiceTemplateSupport:false EnableObservability:false EnableNetworkQoS:false AdvertisedUDNIsolationMode:strict EnableDynamicUDNAllocation:false UDNDeletionGracePeriod:2m0s} Kubernetes:{BootstrapKubeconfig: CertDir: CertDuration:10m0s Kubeconfig: CACert: CACertData: CAData:[] APIServer:https://api.kx-73d77e818b.hypershift.local:443 Token: TokenFile: CompatServiceCIDR: RawServiceCIDRs:172.30.0.0/16 ServiceCIDRs:[] OVNConfigNamespace:openshift-ovn-kubernetes OVNEmptyLbEvents:false RawNoHostSubnetNodes: NoHostSubnetNodes: HostNetworkNamespace:openshift-host-network DisableRequestedChassis:false PlatformType:AWS HealthzBindAddress:0.0.0.0:10256 CompatMetricsBindAddress: CompatOVNMetricsBindAddress: CompatMetricsEnablePprof:false DNSServiceNamespace:openshift-dns DNSServiceName:dns-default} Metrics:{BindAddress: OVNMetricsBindAddress: ExportOVSMetrics:false EnablePprof:false NodeServerPrivKey: NodeServerCert: EnableConfigDuration:false EnableScaleMetrics:false} OvnNorth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnnb_db.db exec:} OvnSouth:{Address: PrivKey: Cert: CACert: CertCommonName: Scheme: ElectionTimer:0 northbound:false RunDir:/var/run/ovn/ DbLocation:/etc/ovn/ovnsb_db.db exec:} Gateway:{Mode:shared Interface: GatewayAcceleratedInterface: EgressGWInterface: NextHop: VLANID:0 NodeportEnable:true DisableSNATMultipleGWs:false V4JoinSubnet:100.64.0.0/16 V6JoinSubnet:fd98::/64 V4MasqueradeSubnet:169.254.169.0/29 V6MasqueradeSubnet:fd69::/125 MasqueradeIPs:{V4OVNMasqueradeIP:169.254.169.1 V6OVNMasqueradeIP:fd69::1 V4HostMasqueradeIP:169.254.169.2 V6HostMasqueradeIP:fd69::2 V4HostETPLocalMasqueradeIP:169.254.169.3 V6HostETPLocalMasqueradeIP:fd69::3 V4DummyNextHopMasqueradeIP:169.254.169.4 V6DummyNextHopMasqueradeIP:fd69::4 V4OVNServiceHairpinMasqueradeIP:169.254.169.5 V6OVNServiceHairpinMasqueradeIP:fd69::5} DisablePacketMTUCheck:false RouterSubnet: SingleNode:false DisableForwarding:false AllowNoUplink:false EphemeralPortRange:} MasterHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} ClusterMgrHA:{ElectionLeaseDuration:137 ElectionRenewDeadline:107 ElectionRetryPeriod:26} HybridOverlay:{Enabled:false RawClusterSubnets: ClusterSubnets:[] VXLANPort:4789} OvnKubeNode:{Mode:full MgmtPortNetdev: MgmtPortDPResourceName:} ClusterManager:{V4TransitSubnet:100.88.0.0/16 V6TransitSubnet:fd97::/64} OvsPaths:{RunDir:/var/run/openvswitch/} NoOverlay:{OutboundSNAT: Routing:} ManagedBGP:{ASNumber:64512 Topology:}} I0429 06:48:16.957983 3947 kube.go:431] Waiting for certificate I0429 06:48:16.958035 3947 certificate_manager.go:422] "Certificate rotation is enabled" logger="kubernetes.io/kube-apiserver-client" I0429 06:48:16.958105 3947 certificate_manager.go:566] "Rotating certificates" logger="kubernetes.io/kube-apiserver-client" I0429 06:48:16.958961 3947 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0429 06:48:16.992915 3947 reflector.go:358] "Starting reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 06:48:16.992936 3947 reflector.go:404] "Listing and watching" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 06:48:16.995679 3947 reflector.go:436] "Caches populated" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 06:48:16.995764 3947 csr.go:274] "Certificate signing request is approved, waiting to be issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-5dr7q" I0429 06:48:17.002926 3947 csr.go:270] "Certificate signing request is issued" logger="kubernetes.io/kube-apiserver-client" csr="csr-5dr7q" I0429 06:48:17.002979 3947 reflector.go:364] "Stopping reflector" logger="kubernetes.io/kube-apiserver-client" type="*v1.CertificateSigningRequest" resyncPeriod="0s" reflector="k8s.io/client-go/tools/watch/informerwatcher.go:162" I0429 06:48:17.958704 3947 kube.go:438] Certificate found I0429 06:48:17.959368 3947 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache" I0429 06:48:17.960675 3947 metrics.go:532] Starting metrics server at address "127.0.0.1:29103" I0429 06:48:18.008838 3947 certificate_manager.go:715] "Certificate rotation deadline determined" logger="kubernetes.io/kube-apiserver-client" expiration="2026-04-30 06:43:16 +0000 UTC" deadline="2026-04-30 03:14:39.362674991 +0000 UTC" I0429 06:48:18.008874 3947 certificate_manager.go:431] "Waiting for next certificate rotation" logger="kubernetes.io/kube-apiserver-client" sleep="20h26m21.35381409s" I0429 06:48:18.027306 3947 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0429 06:48:18.036564 3947 node_controller_manager.go:338] Starting the node network controller manager, Mode: full I0429 06:48:18.036638 3947 factory.go:561] Starting watch factory I0429 06:48:18.037341 3947 reflector.go:358] "Starting reflector" type="*v1.EndpointSlice" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.037368 3947 reflector.go:404] "Listing and watching" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.037439 3947 reflector.go:358] "Starting reflector" type="*v1.Node" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.037465 3947 reflector.go:404] "Listing and watching" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.038475 3947 metrics.go:512] Create OVN Metrics Server on address: 127.0.0.1:29105 I0429 06:48:18.038574 3947 server.go:101] MetricServer registers OVS metrics I0429 06:48:18.038675 3947 reflector.go:358] "Starting reflector" type="*v1.Pod" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.038917 3947 reflector.go:358] "Starting reflector" type="*v1.Namespace" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.039057 3947 reflector.go:404] "Listing and watching" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.039082 3947 reflector.go:404] "Listing and watching" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.039188 3947 reflector.go:358] "Starting reflector" type="*v1.NetworkPolicy" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.039355 3947 reflector.go:404] "Listing and watching" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.039198 3947 server.go:105] MetricServer registers OVN DB metrics I0429 06:48:18.040302 3947 reflector.go:358] "Starting reflector" type="*v1.Service" resyncPeriod="0s" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.040496 3947 reflector.go:404] "Listing and watching" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.044945 3947 ovn_db.go:330] /var/run/openvswitch/ovnnb_db.sock getting info failed: stat /var/run/openvswitch/ovnnb_db.sock: no such file or directory I0429 06:48:18.045029 3947 ovn_db.go:327] ovnnb_db.sock found at /var/run/ovn/ I0429 06:48:18.052061 3947 controller_manager.go:370] Starting the ovnkube controller I0429 06:48:18.052077 3947 controller_manager.go:375] Waiting up to 5m0s for NBDB zone to match: ip-10-0-2-36.ec2.internal I0429 06:48:18.052135 3947 controller_manager.go:395] NBDB zone sync took: 48.005µs I0429 06:48:18.052147 3947 factory.go:561] Starting watch factory I0429 06:48:18.056531 3947 reflector.go:436] "Caches populated" type="*v1.NetworkPolicy" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.056694 3947 reflector.go:436] "Caches populated" type="*v1.EndpointSlice" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.058506 3947 reflector.go:436] "Caches populated" type="*v1.Node" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.060488 3947 reflector.go:436] "Caches populated" type="*v1.Service" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.061184 3947 reflector.go:436] "Caches populated" type="*v1.Namespace" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.063396 3947 ovn_db.go:388] Found OVN NB DB: &{0x1e9e6e0 /etc/ovn/ovnnb_db.db OVN_Northbound 0} I0429 06:48:18.063497 3947 ovn_db.go:395] Found OVN SB DB: &{0x1e9e9a0 /etc/ovn/ovnsb_db.db OVN_Southbound 0} I0429 06:48:18.068368 3947 ovn_db.go:410] Found db is standalone, don't register db_cluster metrics I0429 06:48:18.068406 3947 server.go:109] MetricServer registers OVN Controller metrics I0429 06:48:18.070392 3947 reflector.go:436] "Caches populated" type="*v1.Pod" reflector="k8s.io/client-go/informers/factory.go:160" I0429 06:48:18.073838 3947 server.go:113] MetricServer registers OVN Northd metrics I0429 06:48:18.079124 3947 metrics.go:519] OVN Metrics Server starts to run ... I0429 06:48:18.142870 3947 factory.go:1977] *v1.Namespace informer cache synced successfully I0429 06:48:18.142887 3947 factory.go:1977] *v1.Node informer cache synced successfully I0429 06:48:18.142891 3947 factory.go:1977] *v1.Service informer cache synced successfully I0429 06:48:18.142894 3947 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0429 06:48:18.142897 3947 factory.go:1977] *v1.Pod informer cache synced successfully I0429 06:48:18.142900 3947 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0429 06:48:18.142972 3947 reflector.go:358] "Starting reflector" type="*v1alpha1.BaselineAdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 06:48:18.142985 3947 reflector.go:404] "Listing and watching" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 06:48:18.143016 3947 reflector.go:358] "Starting reflector" type="*v1alpha1.AdminNetworkPolicy" resyncPeriod="0s" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 06:48:18.143045 3947 reflector.go:404] "Listing and watching" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 06:48:18.145888 3947 reflector.go:436] "Caches populated" type="*v1alpha1.AdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 06:48:18.145888 3947 reflector.go:436] "Caches populated" type="*v1alpha1.BaselineAdminNetworkPolicy" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" I0429 06:48:18.148127 3947 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0429 06:48:18.148137 3947 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0429 06:48:18.148208 3947 reflector.go:358] "Starting reflector" type="*v1.EgressIP" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.148232 3947 reflector.go:404] "Listing and watching" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.150504 3947 reflector.go:436] "Caches populated" type="*v1.EgressIP" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.153743 3947 factory.go:1977] *v1.EgressIP informer cache synced successfully I0429 06:48:18.153794 3947 reflector.go:358] "Starting reflector" type="*v1.EgressFirewall" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.153809 3947 reflector.go:404] "Listing and watching" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.156188 3947 reflector.go:436] "Caches populated" type="*v1.EgressFirewall" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.158178 3947 factory.go:1977] *v1.Node informer cache synced successfully I0429 06:48:18.158189 3947 factory.go:1977] *v1.Service informer cache synced successfully I0429 06:48:18.158192 3947 factory.go:1977] *v1.EndpointSlice informer cache synced successfully I0429 06:48:18.158195 3947 factory.go:1977] *v1.Pod informer cache synced successfully I0429 06:48:18.158198 3947 factory.go:1977] *v1.NetworkPolicy informer cache synced successfully I0429 06:48:18.158201 3947 factory.go:1977] *v1.Namespace informer cache synced successfully I0429 06:48:18.159395 3947 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0429 06:48:18.159446 3947 reflector.go:358] "Starting reflector" type="*v1.EgressQoS" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.159457 3947 reflector.go:404] "Listing and watching" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.162090 3947 reflector.go:436] "Caches populated" type="*v1.EgressQoS" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.164106 3947 factory.go:1977] *v1alpha1.AdminNetworkPolicy informer cache synced successfully I0429 06:48:18.164116 3947 factory.go:1977] *v1alpha1.BaselineAdminNetworkPolicy informer cache synced successfully I0429 06:48:18.165343 3947 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0429 06:48:18.165397 3947 reflector.go:358] "Starting reflector" type="*v1.EgressService" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.165409 3947 reflector.go:404] "Listing and watching" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.167796 3947 reflector.go:436] "Caches populated" type="*v1.EgressService" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.169825 3947 factory.go:1977] *v1.EgressIP informer cache synced successfully I0429 06:48:18.171038 3947 factory.go:1977] *v1.EgressService informer cache synced successfully I0429 06:48:18.171089 3947 reflector.go:358] "Starting reflector" type="*v1.AdminPolicyBasedExternalRoute" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.171100 3947 reflector.go:404] "Listing and watching" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.173643 3947 reflector.go:436] "Caches populated" type="*v1.AdminPolicyBasedExternalRoute" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.175612 3947 factory.go:1977] *v1.EgressFirewall informer cache synced successfully I0429 06:48:18.176763 3947 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0429 06:48:18.176895 3947 reflector.go:358] "Starting reflector" type="*v1.NetworkAttachmentDefinition" resyncPeriod="0s" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0429 06:48:18.176906 3947 reflector.go:404] "Listing and watching" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0429 06:48:18.179319 3947 reflector.go:436] "Caches populated" type="*v1.NetworkAttachmentDefinition" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" I0429 06:48:18.181317 3947 factory.go:1977] *v1.EgressQoS informer cache synced successfully I0429 06:48:18.182458 3947 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0429 06:48:18.182514 3947 reflector.go:358] "Starting reflector" type="*v1.ClusterUserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.182528 3947 reflector.go:404] "Listing and watching" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.182604 3947 reflector.go:358] "Starting reflector" type="*v1.UserDefinedNetwork" resyncPeriod="0s" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.182622 3947 reflector.go:404] "Listing and watching" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.185408 3947 reflector.go:436] "Caches populated" type="*v1.UserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.185641 3947 reflector.go:436] "Caches populated" type="*v1.ClusterUserDefinedNetwork" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" I0429 06:48:18.187304 3947 factory.go:1977] *v1.EgressService informer cache synced successfully I0429 06:48:18.187649 3947 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0429 06:48:18.187660 3947 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0429 06:48:18.187667 3947 factory.go:680] Watch Factory start up complete, took: 151.032317ms I0429 06:48:18.187790 3947 default_node_network_controller.go:171] Enable node proxy healthz server on 0.0.0.0:10256 I0429 06:48:18.192513 3947 factory.go:1977] *v1.AdminPolicyBasedExternalRoute informer cache synced successfully I0429 06:48:18.198212 3947 factory.go:1977] *v1.NetworkAttachmentDefinition informer cache synced successfully I0429 06:48:18.203268 3947 factory.go:1977] *v1.UserDefinedNetwork informer cache synced successfully I0429 06:48:18.203281 3947 factory.go:1977] *v1.ClusterUserDefinedNetwork informer cache synced successfully I0429 06:48:18.203288 3947 factory.go:680] Watch Factory start up complete, took: 151.140279ms I0429 06:48:18.203295 3947 controller_manager.go:406] Waiting up to 5m0s for a node to have "ip-10-0-2-36.ec2.internal" zone I0429 06:48:18.239504 3947 default_node_network_controller.go:710] Initializing the default node network controller I0429 06:48:18.327572 3947 udn_isolation.go:101] Starting UDN host isolation manager I0429 06:48:18.334143 3947 udn_isolation.go:112] Found kubelet cgroup path: system.slice/kubelet.service I0429 06:48:18.396837 3947 controller.go:133] Adding controller udn-host-isolation-manager event handlers I0429 06:48:18.396942 3947 shared_informer.go:349] "Waiting for caches to sync" controller="udn-host-isolation-manager" I0429 06:48:18.396962 3947 shared_informer.go:356] "Caches are synced" controller="udn-host-isolation-manager" I0429 06:48:18.432412 3947 controller.go:157] Starting controller udn-host-isolation-manager with 1 workers I0429 06:48:18.432472 3947 default_node_network_controller.go:809] Node ip-10-0-2-36.ec2.internal ready for ovn initialization with subnet 10.129.0.0/23 I0429 06:48:18.437641 3947 kube.go:131] Setting annotations map[k8s.ovn.org/node-encap-ips:["10.0.2.36"] k8s.ovn.org/zone-name:ip-10-0-2-36.ec2.internal] on node ip-10-0-2-36.ec2.internal I0429 06:48:18.459242 3947 gateway_init.go:234] Initializing Gateway Functionality for Gateway PreStart I0429 06:48:18.465805 3947 helper_linux.go:93] Provided gateway interface "br-ex", found as index: 5 I0429 06:48:18.465941 3947 helper_linux.go:118] Found default gateway interface br-ex 10.0.2.1 I0429 06:48:18.466069 3947 gateway_init.go:263] Preparing Gateway I0429 06:48:18.466081 3947 gateway_shared_intf.go:1687] Creating new gateway I0429 06:48:18.575289 3947 iptables.go:108] Creating table: filter chain: FORWARD I0429 06:48:18.582187 3947 iptables.go:108] Creating table: filter chain: OUTPUT I0429 06:48:18.589149 3947 gateway_shared_intf.go:1816] Gateway Creation Complete I0429 06:48:18.589164 3947 kube.go:131] Setting annotations map[k8s.ovn.org/gateway-mtu-support: k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-36.ec2.internal","mac-address":"0e:da:ef:17:b2:1d","ip-addresses":["10.0.2.36/24"],"ip-address":"10.0.2.36/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3a75030f-bcc4-42b7-9f55-c26572786127 k8s.ovn.org/node-encap-ips:["10.0.2.36"] k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.36/24"} k8s.ovn.org/zone-name:ip-10-0-2-36.ec2.internal] on node ip-10-0-2-36.ec2.internal I0429 06:48:18.609075 3947 default_node_network_controller.go:519] Node connection status = connected I0429 06:48:18.703837 3947 controller_manager.go:429] Waiting for node in zone sync took: 500.526717ms I0429 06:48:18.712103 3947 controller_manager.go:611] Switching to transit router for layer2 networks I0429 06:48:18.712116 3947 kube.go:131] Setting annotations map[k8s.ovn.org/layer2-topology-version:2.0] on node ip-10-0-2-36.ec2.internal I0429 06:48:18.727796 3947 metrics.go:164] OVN Kube log file not specified in config, therefore not starting the log file metric monitor I0429 06:48:18.734246 3947 controller_manager.go:290] SCTP support detected in OVN I0429 06:48:18.734813 3947 services_controller.go:69] Creating services controller for network=default I0429 06:48:18.734867 3947 udn_enabled_svc.go:80] Waiting for service informer to sync I0429 06:48:18.734953 3947 controller.go:133] Adding controller [zone-nad-controller NAD controller] event handlers I0429 06:48:18.734983 3947 shared_informer.go:349] "Waiting for caches to sync" controller="[zone-nad-controller NAD controller]" I0429 06:48:18.734993 3947 shared_informer.go:356] "Caches are synced" controller="[zone-nad-controller NAD controller]" I0429 06:48:18.735001 3947 nad_controller.go:541] [zone-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0429 06:48:18.735153 3947 controller.go:157] Starting controller [zone-nad-controller NAD controller] with 1 workers I0429 06:48:18.735192 3947 network_controller.go:373] [zone-nad-controller network controller]: syncing all networks I0429 06:48:18.735254 3947 nad_controller.go:580] [zone-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 66.11µs I0429 06:48:18.735282 3947 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 80.52µs I0429 06:48:18.735294 3947 network_controller.go:384] [zone-nad-controller network controller]: finished syncing all networks. Time taken: 99.718µs I0429 06:48:18.735302 3947 controller.go:157] Starting controller [zone-nad-controller network controller] with 1 workers I0429 06:48:18.735310 3947 nad_controller.go:379] [zone-nad-controller NAD controller]: started I0429 06:48:18.735322 3947 default_network_controller.go:325] Starting the default network controller I0429 06:48:18.735346 3947 udn_enabled_svc.go:95] Performing full resync I0429 06:48:18.735401 3947 network_controller.go:402] [zone-nad-controller network controller]: finished syncing network default, took 4.445µs I0429 06:48:18.735683 3947 udn_enabled_svc.go:99] Waiting for handler to sync I0429 06:48:18.735697 3947 udn_enabled_svc.go:104] Starting worker I0429 06:48:18.744188 3947 default_network_controller.go:395] Cleaning External Gateway ECMP routes I0429 06:48:18.744240 3947 repair.go:33] Syncing exgw routes took 40.021µs I0429 06:48:18.744261 3947 default_network_controller.go:414] Starting all the Watchers... I0429 06:48:18.744437 3947 namespace.go:98] [openshift-backplane-cee] adding namespace I0429 06:48:18.744442 3947 namespace.go:98] [openshift-infra] adding namespace I0429 06:48:18.744448 3947 namespace.go:98] [openshift-user-workload-monitoring] adding namespace I0429 06:48:18.744453 3947 namespace.go:98] [openshift-console] adding namespace I0429 06:48:18.744458 3947 namespace.go:98] [openshift-customer-monitoring] adding namespace I0429 06:48:18.744461 3947 namespace.go:98] [openshift-etcd] adding namespace I0429 06:48:18.744460 3947 namespace.go:98] [openshift-cluster-node-tuning-operator] adding namespace I0429 06:48:18.744472 3947 namespace.go:98] [openshift-backplane-csm] adding namespace I0429 06:48:18.744473 3947 namespace.go:98] [openshift-authentication-operator] adding namespace I0429 06:48:18.744475 3947 namespace.go:98] [openshift-kube-scheduler] adding namespace I0429 06:48:18.744485 3947 namespace.go:98] [openshift] adding namespace I0429 06:48:18.744486 3947 namespace.go:98] [dedicated-admin] adding namespace I0429 06:48:18.744512 3947 namespace.go:98] [openshift-kube-controller-manager] adding namespace I0429 06:48:18.744439 3947 namespace.go:98] [openshift-cloud-network-config-controller] adding namespace I0429 06:48:18.744571 3947 namespace.go:98] [openshift-authentication] adding namespace I0429 06:48:18.745052 3947 namespace.go:102] [openshift-backplane-cee] adding namespace took 603.29µs I0429 06:48:18.745068 3947 namespace.go:98] [openshift-dns] adding namespace I0429 06:48:18.745512 3947 namespace.go:102] [openshift-dns] adding namespace took 438.578µs I0429 06:48:18.745526 3947 namespace.go:98] [openshift-network-console] adding namespace I0429 06:48:18.746020 3947 namespace.go:102] [openshift-network-console] adding namespace took 487.86µs I0429 06:48:18.746034 3947 namespace.go:98] [openshift-operators] adding namespace I0429 06:48:18.746471 3947 namespace.go:102] [openshift-infra] adding namespace took 2.019356ms I0429 06:48:18.746490 3947 namespace.go:98] [openshift-config-operator] adding namespace I0429 06:48:18.746964 3947 namespace.go:102] [openshift-user-workload-monitoring] adding namespace took 2.507082ms I0429 06:48:18.746979 3947 namespace.go:98] [openshift-operator-lifecycle-manager] adding namespace I0429 06:48:18.747765 3947 namespace.go:102] [openshift-console] adding namespace took 3.299629ms I0429 06:48:18.747782 3947 namespace.go:98] [openshift-kube-scheduler-operator] adding namespace I0429 06:48:18.748309 3947 namespace.go:102] [openshift-customer-monitoring] adding namespace took 3.838973ms I0429 06:48:18.748326 3947 namespace.go:98] [openshift-cluster-version] adding namespace I0429 06:48:18.748923 3947 namespace.go:102] [openshift-etcd] adding namespace took 4.451662ms I0429 06:48:18.748941 3947 namespace.go:98] [openshift-marketplace] adding namespace I0429 06:48:18.749468 3947 namespace.go:102] [openshift-cluster-node-tuning-operator] adding namespace took 4.995968ms I0429 06:48:18.749483 3947 namespace.go:98] [openshift-kube-apiserver-operator] adding namespace I0429 06:48:18.750156 3947 namespace.go:102] [openshift-authentication-operator] adding namespace took 5.671913ms I0429 06:48:18.750176 3947 namespace.go:98] [kube-public] adding namespace I0429 06:48:18.750696 3947 namespace.go:102] [openshift-backplane-csm] adding namespace took 6.216884ms I0429 06:48:18.750712 3947 namespace.go:98] [openshift-deployment-validation-operator] adding namespace I0429 06:48:18.751298 3947 namespace.go:102] [openshift-kube-scheduler] adding namespace took 6.814289ms I0429 06:48:18.751313 3947 namespace.go:98] [openshift-ingress-canary] adding namespace I0429 06:48:18.751849 3947 namespace.go:102] [openshift] adding namespace took 7.358616ms I0429 06:48:18.751865 3947 namespace.go:98] [openshift-logging] adding namespace I0429 06:48:18.752383 3947 namespace.go:102] [dedicated-admin] adding namespace took 7.88656ms I0429 06:48:18.752397 3947 namespace.go:98] [openshift-cluster-machine-approver] adding namespace I0429 06:48:18.752950 3947 namespace.go:102] [openshift-kube-controller-manager] adding namespace took 8.428071ms I0429 06:48:18.752967 3947 namespace.go:98] [default] adding namespace I0429 06:48:18.753482 3947 namespace.go:102] [openshift-cloud-network-config-controller] adding namespace took 8.929237ms I0429 06:48:18.753497 3947 namespace.go:98] [openshift-network-operator] adding namespace I0429 06:48:18.754008 3947 namespace.go:102] [openshift-authentication] adding namespace took 9.426033ms I0429 06:48:18.754020 3947 namespace.go:98] [openshift-machine-config-operator] adding namespace I0429 06:48:18.754535 3947 namespace.go:102] [openshift-operators] adding namespace took 8.495088ms I0429 06:48:18.754550 3947 namespace.go:98] [openshift-backplane-mcs-tier-two] adding namespace I0429 06:48:18.754991 3947 namespace.go:102] [openshift-config-operator] adding namespace took 8.493178ms I0429 06:48:18.755009 3947 namespace.go:98] [openshift-backplane-lpsre] adding namespace I0429 06:48:18.755481 3947 namespace.go:102] [openshift-operator-lifecycle-manager] adding namespace took 8.495659ms I0429 06:48:18.755495 3947 namespace.go:98] [openshift-cluster-samples-operator] adding namespace I0429 06:48:18.756055 3947 namespace.go:102] [openshift-kube-scheduler-operator] adding namespace took 8.265373ms I0429 06:48:18.756072 3947 namespace.go:98] [openshift-insights] adding namespace I0429 06:48:18.756451 3947 namespace.go:102] [openshift-cluster-version] adding namespace took 8.119959ms I0429 06:48:18.756464 3947 namespace.go:98] [openshift-network-diagnostics] adding namespace I0429 06:48:18.756896 3947 namespace.go:102] [openshift-marketplace] adding namespace took 7.947104ms I0429 06:48:18.756916 3947 namespace.go:98] [openshift-kube-apiserver] adding namespace I0429 06:48:18.757405 3947 namespace.go:102] [openshift-kube-apiserver-operator] adding namespace took 7.914855ms I0429 06:48:18.757419 3947 namespace.go:98] [openshift-backplane-mobb] adding namespace I0429 06:48:18.757882 3947 namespace.go:102] [kube-public] adding namespace took 7.699853ms I0429 06:48:18.757899 3947 namespace.go:98] [openshift-machine-api] adding namespace I0429 06:48:18.758485 3947 namespace.go:102] [openshift-deployment-validation-operator] adding namespace took 7.765966ms I0429 06:48:18.758507 3947 namespace.go:98] [openshift-multus] adding namespace I0429 06:48:18.759054 3947 namespace.go:102] [openshift-ingress-canary] adding namespace took 7.733386ms I0429 06:48:18.759082 3947 namespace.go:98] [openshift-console-operator] adding namespace I0429 06:48:18.759482 3947 namespace.go:102] [openshift-logging] adding namespace took 7.609771ms I0429 06:48:18.759502 3947 namespace.go:98] [openshift-must-gather-operator] adding namespace I0429 06:48:18.760132 3947 namespace.go:102] [openshift-cluster-machine-approver] adding namespace took 7.728573ms I0429 06:48:18.760151 3947 namespace.go:98] [openshift-backplane-cse] adding namespace I0429 06:48:18.760558 3947 namespace.go:102] [default] adding namespace took 7.585759ms I0429 06:48:18.760572 3947 namespace.go:98] [openshift-backplane-srep-ro] adding namespace I0429 06:48:18.761142 3947 namespace.go:102] [openshift-network-operator] adding namespace took 7.635067ms I0429 06:48:18.761162 3947 namespace.go:98] [openshift-image-registry] adding namespace I0429 06:48:18.761780 3947 namespace.go:102] [openshift-machine-config-operator] adding namespace took 7.75412ms I0429 06:48:18.761799 3947 namespace.go:98] [openshift-host-network] adding namespace I0429 06:48:18.762279 3947 namespace.go:102] [openshift-backplane-mcs-tier-two] adding namespace took 7.723531ms I0429 06:48:18.762295 3947 namespace.go:98] [openshift-ingress] adding namespace I0429 06:48:18.762843 3947 namespace.go:102] [openshift-backplane-lpsre] adding namespace took 7.827759ms I0429 06:48:18.762861 3947 namespace.go:98] [openshift-operators-redhat] adding namespace I0429 06:48:18.763328 3947 namespace.go:102] [openshift-cluster-samples-operator] adding namespace took 7.827994ms I0429 06:48:18.763342 3947 namespace.go:98] [open-cluster-management-agent-addon] adding namespace I0429 06:48:18.763695 3947 namespace.go:102] [openshift-insights] adding namespace took 7.616947ms I0429 06:48:18.763710 3947 namespace.go:98] [openshift-controller-manager-operator] adding namespace W0429 06:48:18.763723 3947 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-88f5c: pod openshift-network-diagnostics/network-check-target-88f5c: no pod IPs found W0429 06:48:18.763738 3947 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-network-diagnostics/network-check-target-mkl5n: pod openshift-network-diagnostics/network-check-target-mkl5n: no pod IPs found I0429 06:48:18.764231 3947 namespace.go:102] [openshift-network-diagnostics] adding namespace took 7.76196ms I0429 06:48:18.764244 3947 namespace.go:98] [openshift-node] adding namespace I0429 06:48:18.764568 3947 namespace.go:102] [openshift-kube-apiserver] adding namespace took 7.646742ms I0429 06:48:18.764598 3947 namespace.go:98] [openshift-backplane-managed-scripts] adding namespace I0429 06:48:18.765052 3947 namespace.go:102] [openshift-backplane-mobb] adding namespace took 7.628071ms I0429 06:48:18.765063 3947 namespace.go:98] [openshift-route-controller-manager] adding namespace W0429 06:48:18.765613 3947 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-s9ckh: pod openshift-multus/network-metrics-daemon-s9ckh: no pod IPs found I0429 06:48:18.765643 3947 namespace.go:102] [openshift-machine-api] adding namespace took 7.736963ms W0429 06:48:18.765645 3947 base_network_controller_namespace.go:410] Failed to get IPs for pod openshift-multus/network-metrics-daemon-764nj: pod openshift-multus/network-metrics-daemon-764nj: no pod IPs found I0429 06:48:18.765659 3947 namespace.go:98] [openshift-network-node-identity] adding namespace I0429 06:48:18.766117 3947 namespace.go:102] [openshift-multus] adding namespace took 7.605576ms I0429 06:48:18.766127 3947 namespace.go:98] [openshift-kube-storage-version-migrator-operator] adding namespace I0429 06:48:18.766614 3947 namespace.go:102] [openshift-console-operator] adding namespace took 7.524071ms I0429 06:48:18.766630 3947 namespace.go:98] [openshift-backplane] adding namespace I0429 06:48:18.767066 3947 namespace.go:102] [openshift-must-gather-operator] adding namespace took 7.558687ms I0429 06:48:18.767078 3947 namespace.go:98] [kube-system] adding namespace I0429 06:48:18.767501 3947 namespace.go:102] [openshift-backplane-cse] adding namespace took 7.343367ms I0429 06:48:18.767512 3947 namespace.go:98] [openshift-cloud-controller-manager] adding namespace I0429 06:48:18.767966 3947 namespace.go:102] [openshift-backplane-srep-ro] adding namespace took 7.389162ms I0429 06:48:18.767975 3947 namespace.go:98] [openshift-ovn-kubernetes] adding namespace I0429 06:48:18.768420 3947 namespace.go:102] [openshift-image-registry] adding namespace took 7.253363ms I0429 06:48:18.768429 3947 namespace.go:98] [openshift-package-operator] adding namespace I0429 06:48:18.768978 3947 namespace.go:102] [openshift-host-network] adding namespace took 7.17034ms I0429 06:48:18.769004 3947 namespace.go:98] [openshift-backplane-srep] adding namespace I0429 06:48:18.769605 3947 namespace.go:102] [openshift-ingress] adding namespace took 7.303419ms I0429 06:48:18.770076 3947 namespace.go:102] [openshift-operators-redhat] adding namespace took 7.208867ms I0429 06:48:18.770090 3947 namespace.go:98] [openshift-kube-controller-manager-operator] adding namespace I0429 06:48:18.770530 3947 namespace.go:102] [open-cluster-management-agent-addon] adding namespace took 7.18351ms I0429 06:48:18.770539 3947 namespace.go:98] [openshift-monitoring] adding namespace I0429 06:48:18.771051 3947 namespace.go:102] [openshift-controller-manager-operator] adding namespace took 7.335939ms I0429 06:48:18.771061 3947 namespace.go:98] [openshift-apiserver] adding namespace I0429 06:48:18.771530 3947 namespace.go:102] [openshift-node] adding namespace took 7.279895ms I0429 06:48:18.771547 3947 namespace.go:98] [openshift-controller-manager] adding namespace I0429 06:48:18.771951 3947 namespace.go:102] [openshift-backplane-managed-scripts] adding namespace took 7.347827ms I0429 06:48:18.771965 3947 namespace.go:98] [openshift-console-user-settings] adding namespace I0429 06:48:18.772408 3947 namespace.go:102] [openshift-route-controller-manager] adding namespace took 7.340714ms I0429 06:48:18.772419 3947 namespace.go:98] [openshift-config-managed] adding namespace I0429 06:48:18.773455 3947 namespace.go:102] [openshift-network-node-identity] adding namespace took 7.791726ms I0429 06:48:18.773464 3947 namespace.go:98] [openshift-backplane-tam] adding namespace I0429 06:48:18.773989 3947 namespace.go:102] [openshift-kube-storage-version-migrator-operator] adding namespace took 7.85866ms I0429 06:48:18.774000 3947 namespace.go:98] [openshift-service-ca-operator] adding namespace I0429 06:48:18.774552 3947 namespace.go:102] [openshift-backplane] adding namespace took 7.916142ms I0429 06:48:18.774566 3947 namespace.go:98] [openshift-cluster-csi-drivers] adding namespace I0429 06:48:18.775019 3947 namespace.go:102] [kube-system] adding namespace took 7.936661ms I0429 06:48:18.775030 3947 namespace.go:98] [openshift-apiserver-operator] adding namespace I0429 06:48:18.775432 3947 namespace.go:102] [openshift-cloud-controller-manager] adding namespace took 7.91451ms I0429 06:48:18.775441 3947 namespace.go:98] [kube-node-lease] adding namespace I0429 06:48:18.776017 3947 namespace.go:102] [openshift-ovn-kubernetes] adding namespace took 8.038538ms I0429 06:48:18.776028 3947 namespace.go:98] [openshift-cloud-credential-operator] adding namespace I0429 06:48:18.776521 3947 namespace.go:102] [openshift-package-operator] adding namespace took 8.086404ms I0429 06:48:18.776533 3947 namespace.go:98] [openshift-dns-operator] adding namespace I0429 06:48:18.776930 3947 namespace.go:102] [openshift-backplane-srep] adding namespace took 7.921467ms I0429 06:48:18.776939 3947 namespace.go:98] [openshift-config] adding namespace I0429 06:48:18.777550 3947 namespace.go:102] [openshift-kube-controller-manager-operator] adding namespace took 7.451619ms I0429 06:48:18.777570 3947 namespace.go:98] [openshift-ingress-operator] adding namespace I0429 06:48:18.778004 3947 namespace.go:102] [openshift-monitoring] adding namespace took 7.45778ms I0429 06:48:18.778555 3947 namespace.go:102] [openshift-apiserver] adding namespace took 7.489788ms I0429 06:48:18.778571 3947 namespace.go:98] [open-cluster-management-2pvlgkdgsf526nqovfd11k6703l08h7f] adding namespace I0429 06:48:18.779125 3947 namespace.go:102] [openshift-controller-manager] adding namespace took 7.571704ms I0429 06:48:18.779519 3947 namespace.go:102] [openshift-console-user-settings] adding namespace took 7.544707ms I0429 06:48:18.780033 3947 namespace.go:102] [openshift-config-managed] adding namespace took 7.607162ms I0429 06:48:18.780675 3947 namespace.go:102] [openshift-backplane-tam] adding namespace took 7.20576ms I0429 06:48:18.780689 3947 namespace.go:98] [openshift-cluster-storage-operator] adding namespace I0429 06:48:18.781175 3947 namespace.go:102] [openshift-service-ca-operator] adding namespace took 7.169866ms I0429 06:48:18.781645 3947 namespace.go:102] [openshift-cluster-csi-drivers] adding namespace took 7.071328ms I0429 06:48:18.782149 3947 namespace.go:102] [openshift-apiserver-operator] adding namespace took 7.113347ms I0429 06:48:18.782598 3947 namespace.go:102] [kube-node-lease] adding namespace took 7.136678ms I0429 06:48:18.783153 3947 namespace.go:102] [openshift-cloud-credential-operator] adding namespace took 7.120236ms I0429 06:48:18.783678 3947 namespace.go:102] [openshift-dns-operator] adding namespace took 7.138211ms I0429 06:48:18.784138 3947 namespace.go:102] [openshift-config] adding namespace took 7.193997ms I0429 06:48:18.784545 3947 namespace.go:102] [openshift-ingress-operator] adding namespace took 6.969902ms I0429 06:48:18.785230 3947 namespace.go:102] [open-cluster-management-2pvlgkdgsf526nqovfd11k6703l08h7f] adding namespace took 6.651378ms I0429 06:48:18.785715 3947 namespace.go:102] [openshift-cluster-storage-operator] adding namespace took 5.02008ms W0429 06:48:18.785797 3947 master.go:471] Unable to parse remote node ip-10-0-0-41.ec2.internal chassis-id annotation. Chassis may be removed during sync I0429 06:48:18.785903 3947 master.go:548] Adding or Updating local node "ip-10-0-2-36.ec2.internal" for network "default" I0429 06:48:18.785966 3947 obj_retry.go:578] Failed to create *v1.Node ip-10-0-0-41.ec2.internal, error: adding or updating remote node chassis ip-10-0-0-41.ec2.internal failed, err - failed to create or update chassis to remote for remote node ip-10-0-0-41.ec2.internal, error: failed to parse node chassis-id for node - ip-10-0-0-41.ec2.internal, error: suppressed error logged: k8s.ovn.org/node-chassis-id annotation not found for node ip-10-0-0-41.ec2.internal I0429 06:48:18.786961 3947 zone_ic_handler.go:195] Time taken to create transit switch: 421.758µs I0429 06:48:18.786978 3947 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-1-61.ec2.internal for the network default I0429 06:48:18.788499 3947 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-1-61.ec2.internal" on network "default" took: 1.971929ms W0429 06:48:18.794362 3947 gateway.go:207] Unable to fetch podIPs for pod openshift-network-diagnostics/network-check-target-mkl5n: pod openshift-network-diagnostics/network-check-target-mkl5n: no pod IPs found W0429 06:48:18.794388 3947 gateway.go:207] Unable to fetch podIPs for pod openshift-multus/network-metrics-daemon-764nj: pod openshift-multus/network-metrics-daemon-764nj: no pod IPs found I0429 06:48:18.795276 3947 zone_ic_handler.go:203] Creating interconnect resources for local zone node ip-10-0-2-36.ec2.internal for the network default I0429 06:48:18.795427 3947 zone_ic_handler.go:195] Time taken to create transit switch: 139.397µs I0429 06:48:18.796271 3947 services_controller.go:192] Starting controller ovn-lb-controller for network=default I0429 06:48:18.796297 3947 services_controller.go:200] Waiting for node tracker handler to sync for network=default I0429 06:48:18.796314 3947 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-41.ec2.internal in network "default" I0429 06:48:18.796355 3947 node_tracker.go:235] Node ip-10-0-0-41.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-41.ec2.internal" I0429 06:48:18.796370 3947 node_tracker.go:169] Node ip-10-0-0-41.ec2.internal switch + router changed, syncing services in network "default" I0429 06:48:18.796383 3947 services_controller.go:574] Full service sync requested for network=default I0429 06:48:18.796388 3947 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-61.ec2.internal in network "default" I0429 06:48:18.796312 3947 shared_informer.go:349] "Waiting for caches to sync" controller="node-tracker-controller" I0429 06:48:18.796423 3947 node_tracker.go:169] Node ip-10-0-1-61.ec2.internal switch + router changed, syncing services in network "default" I0429 06:48:18.796456 3947 services_controller.go:574] Full service sync requested for network=default I0429 06:48:18.796467 3947 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-36.ec2.internal in network "default" I0429 06:48:18.796506 3947 node_tracker.go:169] Node ip-10-0-2-36.ec2.internal switch + router changed, syncing services in network "default" I0429 06:48:18.796514 3947 services_controller.go:574] Full service sync requested for network=default I0429 06:48:18.897520 3947 shared_informer.go:356] "Caches are synced" controller="node-tracker-controller" I0429 06:48:18.897539 3947 services_controller.go:208] Setting up event handlers for services for network=default I0429 06:48:18.897602 3947 services_controller.go:218] Setting up event handlers for endpoint slices for network=default I0429 06:48:18.897662 3947 services_controller.go:233] Waiting for service and endpoint handlers to sync for network=default I0429 06:48:18.897677 3947 shared_informer.go:349] "Waiting for caches to sync" controller="ovn-lb-controller" I0429 06:48:18.998441 3947 shared_informer.go:356] "Caches are synced" controller="ovn-lb-controller" I0429 06:48:18.998464 3947 repair.go:60] Starting repairing loop for services I0429 06:48:18.998597 3947 repair.go:131] Deleted 0 stale service LBs I0429 06:48:18.998627 3947 repair.go:137] Deleted 0 stale Chassis Template Vars I0429 06:48:18.998658 3947 repair.go:62] Finished repairing loop for services: 195.767µs I0429 06:48:18.998693 3947 services_controller.go:363] Controller cache of 0 load balancers initialized for 0 services for network=default I0429 06:48:18.998707 3947 services_controller.go:254] Starting workers for network=default I0429 06:48:18.999296 3947 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-764nj] creating logical port openshift-multus_network-metrics-daemon-764nj for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:18.999295 3947 obj_retry.go:578] Failed to create *v1.Pod openshift-multus/network-metrics-daemon-s9ckh, error: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-s9ckh: suppressed error logged: pod openshift-multus/network-metrics-daemon-s9ckh: no pod IPs found I0429 06:48:18.999322 3947 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-mkl5n] creating logical port openshift-network-diagnostics_network-check-target-mkl5n for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:18.999337 3947 obj_retry.go:578] Failed to create *v1.Pod openshift-network-diagnostics/network-check-target-88f5c, error: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-88f5c: suppressed error logged: pod openshift-network-diagnostics/network-check-target-88f5c: no pod IPs found I0429 06:48:18.999435 3947 kube.go:256] Updating pod openshift-multus/network-metrics-daemon-764nj I0429 06:48:18.999571 3947 kube.go:256] Updating pod openshift-network-diagnostics/network-check-target-mkl5n I0429 06:48:19.016392 3947 pod.go:62] [openshift-network-diagnostics/network-check-target-mkl5n] pod update took 16.997429ms I0429 06:48:19.016409 3947 base_network_controller_pods.go:951] [default/openshift-network-diagnostics/network-check-target-mkl5n] addLogicalPort annotation time took 17.018053ms I0429 06:48:19.017173 3947 pods.go:271] [openshift-network-diagnostics/network-check-target-mkl5n] addLogicalPort took 17.905977ms, libovsdb time 552.675µs I0429 06:48:19.018210 3947 pod.go:62] [openshift-multus/network-metrics-daemon-764nj] pod update took 18.821656ms I0429 06:48:19.018227 3947 base_network_controller_pods.go:951] [default/openshift-multus/network-metrics-daemon-764nj] addLogicalPort annotation time took 18.861267ms I0429 06:48:19.018963 3947 pods.go:271] [openshift-multus/network-metrics-daemon-764nj] addLogicalPort took 19.680297ms, libovsdb time 575.697µs I0429 06:48:19.019474 3947 admin_network_policy_controller.go:227] Starting controller default-network-controller I0429 06:48:19.019496 3947 shared_informer.go:349] "Waiting for caches to sync" controller="default-network-controller" I0429 06:48:19.019509 3947 shared_informer.go:356] "Caches are synced" controller="default-network-controller" I0429 06:48:19.019513 3947 admin_network_policy_controller.go:237] Repairing Admin Network Policies I0429 06:48:19.019698 3947 repair.go:29] Repairing admin network policies took 180.066µs I0429 06:48:19.019798 3947 repair.go:92] Repairing baseline admin network policies took 91.907µs I0429 06:48:19.020152 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-metrics for network default I0429 06:48:19.020171 3947 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-metrics added to peer address sets [] I0429 06:48:19.021663 3947 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-metrics resources completed, update namespace loglevel I0429 06:48:19.021693 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/default-deny for network default I0429 06:48:19.021700 3947 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/default-deny added to peer address sets [] I0429 06:48:19.021990 3947 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/default-deny resources completed, update namespace loglevel I0429 06:48:19.022020 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-egress-to-api-server for network default I0429 06:48:19.022031 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-egress-to-api-server added to peer address sets [] I0429 06:48:19.023444 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-egress-to-api-server resources completed, update namespace loglevel I0429 06:48:19.023464 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range for network default I0429 06:48:19.023473 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range added to peer address sets [] I0429 06:48:19.023970 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-operator-metrics-range resources completed, update namespace loglevel I0429 06:48:19.023990 3947 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-insights for network default I0429 06:48:19.023997 3947 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-insights added to peer address sets [] I0429 06:48:19.025898 3947 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-insights resources completed, update namespace loglevel I0429 06:48:19.025916 3947 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server for network default I0429 06:48:19.025924 3947 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server added to peer address sets [] I0429 06:48:19.026377 3947 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-grpc-server resources completed, update namespace loglevel I0429 06:48:19.026394 3947 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/catalog-operator for network default I0429 06:48:19.026405 3947 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/catalog-operator added to peer address sets [] I0429 06:48:19.028439 3947 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/catalog-operator resources completed, update namespace loglevel I0429 06:48:19.028457 3947 base_network_controller_policy.go:1179] Adding network policy openshift-operators/default-allow-all for network default I0429 06:48:19.028465 3947 base_network_controller_policy.go:1054] Policy openshift-operators/default-allow-all added to peer address sets [] I0429 06:48:19.029982 3947 base_network_controller_policy.go:1232] Create network policy openshift-operators/default-allow-all resources completed, update namespace loglevel I0429 06:48:19.029998 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-webhook for network default I0429 06:48:19.030007 3947 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-webhook added to peer address sets [] I0429 06:48:19.030438 3947 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-webhook resources completed, update namespace loglevel I0429 06:48:19.030456 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-egress for network default I0429 06:48:19.030464 3947 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-egress added to peer address sets [] I0429 06:48:19.031100 3947 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-egress resources completed, update namespace loglevel I0429 06:48:19.031124 3947 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles for network default I0429 06:48:19.031133 3947 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles added to peer address sets [] I0429 06:48:19.031952 3947 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles resources completed, update namespace loglevel I0429 06:48:19.031971 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-to-dns for network default I0429 06:48:19.032321 3947 pod_selector_address_set.go:213] Created shared address set for pod selector LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},} I0429 06:48:19.032335 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0429 06:48:19.033001 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-to-dns resources completed, update namespace loglevel I0429 06:48:19.033020 3947 base_network_controller_policy.go:1179] Adding network policy openshift-deployment-validation-operator/allow-from-openshift-olm for network default I0429 06:48:19.033028 3947 base_network_controller_policy.go:1054] Policy openshift-deployment-validation-operator/allow-from-openshift-olm added to peer address sets [] I0429 06:48:19.033476 3947 base_network_controller_policy.go:1232] Create network policy openshift-deployment-validation-operator/allow-from-openshift-olm resources completed, update namespace loglevel I0429 06:48:19.033491 3947 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/package-server-manager for network default I0429 06:48:19.033501 3947 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/package-server-manager added to peer address sets [] I0429 06:48:19.034669 3947 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/package-server-manager resources completed, update namespace loglevel I0429 06:48:19.034686 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-all-egress for network default I0429 06:48:19.034700 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-all-egress added to peer address sets [] I0429 06:48:19.036168 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-all-egress resources completed, update namespace loglevel I0429 06:48:19.036190 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-egress-to-api-server for network default I0429 06:48:19.036198 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-egress-to-api-server added to peer address sets [] I0429 06:48:19.036686 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-egress-to-api-server resources completed, update namespace loglevel I0429 06:48:19.036703 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-version/default-deny for network default I0429 06:48:19.036710 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-version/default-deny added to peer address sets [] I0429 06:48:19.038053 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-version/default-deny resources completed, update namespace loglevel I0429 06:48:19.038070 3947 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/marketplace-operator for network default I0429 06:48:19.038079 3947 base_network_controller_policy.go:1054] Policy openshift-marketplace/marketplace-operator added to peer address sets [] I0429 06:48:19.040083 3947 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/marketplace-operator resources completed, update namespace loglevel I0429 06:48:19.040100 3947 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/olm-operator for network default I0429 06:48:19.040108 3947 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/olm-operator added to peer address sets [] I0429 06:48:19.041194 3947 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/olm-operator resources completed, update namespace loglevel I0429 06:48:19.041218 3947 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/packageserver for network default I0429 06:48:19.041227 3947 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/packageserver added to peer address sets [] I0429 06:48:19.042468 3947 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/packageserver resources completed, update namespace loglevel I0429 06:48:19.042487 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics for network default I0429 06:48:19.042495 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics added to peer address sets [] I0429 06:48:19.043028 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-ingress-to-operator-metrics resources completed, update namespace loglevel I0429 06:48:19.043048 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/allow-to-dns for network default I0429 06:48:19.043086 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/allow-to-dns added to peer address sets [LS{ML:{kubernetes.io/metadata.name: openshift-dns,},}_LS{ML:{dns.operator.openshift.io/daemonset-dns: default,},}] I0429 06:48:19.043829 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/allow-to-dns resources completed, update namespace loglevel I0429 06:48:19.043846 3947 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/default-deny-all for network default I0429 06:48:19.043852 3947 base_network_controller_policy.go:1054] Policy openshift-marketplace/default-deny-all added to peer address sets [] I0429 06:48:19.044193 3947 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/default-deny-all resources completed, update namespace loglevel I0429 06:48:19.044212 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-storage-operator/default-deny for network default I0429 06:48:19.044218 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-storage-operator/default-deny added to peer address sets [] I0429 06:48:19.044524 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-storage-operator/default-deny resources completed, update namespace loglevel I0429 06:48:19.044542 3947 base_network_controller_policy.go:1179] Adding network policy openshift-operator-lifecycle-manager/default-deny-all-traffic for network default I0429 06:48:19.044551 3947 base_network_controller_policy.go:1054] Policy openshift-operator-lifecycle-manager/default-deny-all-traffic added to peer address sets [] I0429 06:48:19.044869 3947 base_network_controller_policy.go:1232] Create network policy openshift-operator-lifecycle-manager/default-deny-all-traffic resources completed, update namespace loglevel I0429 06:48:19.044887 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cloud-credential-operator/allow-ingress-pprof for network default I0429 06:48:19.044894 3947 base_network_controller_policy.go:1054] Policy openshift-cloud-credential-operator/allow-ingress-pprof added to peer address sets [] I0429 06:48:19.045445 3947 base_network_controller_policy.go:1232] Create network policy openshift-cloud-credential-operator/allow-ingress-pprof resources completed, update namespace loglevel I0429 06:48:19.045461 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-all-egress for network default I0429 06:48:19.045468 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-all-egress added to peer address sets [] I0429 06:48:19.045986 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-all-egress resources completed, update namespace loglevel I0429 06:48:19.046006 3947 base_network_controller_policy.go:1179] Adding network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range for network default I0429 06:48:19.046014 3947 base_network_controller_policy.go:1054] Policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range added to peer address sets [] I0429 06:48:19.046845 3947 base_network_controller_policy.go:1232] Create network policy openshift-cluster-csi-drivers/allow-ingress-to-metrics-range resources completed, update namespace loglevel I0429 06:48:19.046864 3947 base_network_controller_policy.go:1179] Adding network policy openshift-marketplace/unpack-bundles for network default I0429 06:48:19.046886 3947 base_network_controller_policy.go:1054] Policy openshift-marketplace/unpack-bundles added to peer address sets [] I0429 06:48:19.047828 3947 base_network_controller_policy.go:1232] Create network policy openshift-marketplace/unpack-bundles resources completed, update namespace loglevel I0429 06:48:19.052391 3947 egressfirewall.go:374] Starting EgressFirewall controller I0429 06:48:19.052415 3947 controller.go:133] Adding controller egress-firewall-controller-node event handlers I0429 06:48:19.052417 3947 controller.go:133] Adding controller egress-firewall-controller event handlers I0429 06:48:19.052461 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller" I0429 06:48:19.052478 3947 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller" I0429 06:48:19.052466 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egress-firewall-controller-node" I0429 06:48:19.052485 3947 shared_informer.go:356] "Caches are synced" controller="egress-firewall-controller-node" I0429 06:48:19.052761 3947 controller.go:157] Starting controller egress-firewall-controller with 1 workers I0429 06:48:19.052776 3947 controller.go:157] Starting controller egress-firewall-controller-node with 1 workers I0429 06:48:19.052782 3947 controller.go:157] Starting controller egress-firewall-controller-NAD with 1 workers I0429 06:48:19.052792 3947 egressqos.go:193] Setting up event handlers for EgressQoS I0429 06:48:19.052796 3947 egressfirewall.go:723] Syncing node "ip-10-0-0-41.ec2.internal" for egress firewall I0429 06:48:19.052805 3947 egressfirewall.go:723] Syncing node "ip-10-0-1-61.ec2.internal" for egress firewall I0429 06:48:19.052810 3947 egressfirewall.go:723] Syncing node "ip-10-0-2-36.ec2.internal" for egress firewall I0429 06:48:19.052904 3947 egressqos.go:245] Starting EgressQoS Controller I0429 06:48:19.052919 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressqosnodes" I0429 06:48:19.052928 3947 shared_informer.go:356] "Caches are synced" controller="egressqosnodes" I0429 06:48:19.052935 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressqospods" I0429 06:48:19.052942 3947 shared_informer.go:356] "Caches are synced" controller="egressqospods" I0429 06:48:19.052949 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressqos" I0429 06:48:19.052954 3947 shared_informer.go:356] "Caches are synced" controller="egressqos" I0429 06:48:19.052958 3947 egressqos.go:259] Repairing EgressQoSes I0429 06:48:19.052962 3947 egressqos.go:399] Starting repairing loop for egressqos I0429 06:48:19.053044 3947 egressqos.go:401] Finished repairing loop for egressqos: 81.957µs I0429 06:48:19.053061 3947 egressservice_zone.go:132] Setting up event handlers for Egress Services I0429 06:48:19.053105 3947 egressqos.go:1007] Processing sync for EgressQoS node ip-10-0-2-36.ec2.internal I0429 06:48:19.053146 3947 egressqos.go:1010] Finished syncing EgressQoS node ip-10-0-2-36.ec2.internal : 43.047µs I0429 06:48:19.053185 3947 egressservice_zone.go:210] Starting Egress Services Controller I0429 06:48:19.053195 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0429 06:48:19.053201 3947 shared_informer.go:356] "Caches are synced" controller="egressservices" I0429 06:48:19.053206 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0429 06:48:19.053211 3947 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0429 06:48:19.053219 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0429 06:48:19.053226 3947 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0429 06:48:19.053234 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_nodes" I0429 06:48:19.053239 3947 shared_informer.go:356] "Caches are synced" controller="egressservices_nodes" I0429 06:48:19.053243 3947 egressservice_zone.go:228] Repairing Egress Services I0429 06:48:19.053496 3947 master_controller.go:88] Starting Admin Policy Based Route Controller I0429 06:48:19.053504 3947 external_controller.go:278] Starting Admin Policy Based Route Controller I0429 06:48:19.053528 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-36.ec2.internal I0429 06:48:19.053539 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-36.ec2.internal: 14.378µs I0429 06:48:19.053546 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-41.ec2.internal I0429 06:48:19.053550 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-41.ec2.internal: 3.939µs I0429 06:48:19.053554 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-61.ec2.internal I0429 06:48:19.053557 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-61.ec2.internal: 3.399µs I0429 06:48:19.053669 3947 default_network_controller.go:585] Completing all the Watchers took 309.397928ms I0429 06:48:19.053689 3947 default_network_controller.go:589] Starting unidling controllers I0429 06:48:19.053707 3947 unidle.go:46] Registering OVN SB ControllerEvent handler I0429 06:48:19.053720 3947 unidle.go:63] Populating Initial ContollerEvent events I0429 06:48:19.053735 3947 unidle.go:79] Setting up event handlers for services I0429 06:48:19.053810 3947 ovnkube.go:545] Waiting for OVN northbound database changes to be processed by ovn-controller I0429 06:48:19.109978 3947 default_node_network_controller.go:519] Node connection status = connected I0429 06:48:19.609691 3947 default_node_network_controller.go:519] Node connection status = connected I0429 06:48:19.930913 3947 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-1-61.ec2.internal in network "default" I0429 06:48:19.930948 3947 egressfirewall.go:723] Syncing node "ip-10-0-1-61.ec2.internal" for egress firewall I0429 06:48:19.930966 3947 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-1-61.ec2.internal. Updating no re-route policies I0429 06:48:19.930969 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-61.ec2.internal I0429 06:48:19.930973 3947 node_tracker.go:169] Node ip-10-0-1-61.ec2.internal switch + router changed, syncing services in network "default" I0429 06:48:19.930983 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-61.ec2.internal: 18.12µs I0429 06:48:19.930988 3947 services_controller.go:574] Full service sync requested for network=default I0429 06:48:20.110192 3947 default_node_network_controller.go:519] Node connection status = connected I0429 06:48:20.289560 3947 egressfirewall.go:723] Syncing node "ip-10-0-0-41.ec2.internal" for egress firewall I0429 06:48:20.289604 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-41.ec2.internal I0429 06:48:20.289620 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-41.ec2.internal: 40.123µs I0429 06:48:20.289620 3947 default_network_controller.go:1025] Node "ip-10-0-0-41.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 06:48:20.610245 3947 default_node_network_controller.go:519] Node connection status = connected I0429 06:48:20.829338 3947 ovnkube.go:550] Finished waiting for OVN northbound database changes to be processed by ovn-controller I0429 06:48:21.110574 3947 default_node_network_controller.go:519] Node connection status = connected I0429 06:48:21.128858 3947 bridgeconfig.go:537] Gateway is ready I0429 06:48:21.128882 3947 gateway_shared_intf.go:1733] Creating Gateway Openflow Manager I0429 06:48:21.141106 3947 node_ip_handler_linux.go:247] Node primary address changed to 10.0.2.36. Updating OVN encap IP. I0429 06:48:21.146725 3947 node_ip_handler_linux.go:542] Will not update encap IP 10.0.2.36 - it is already configured I0429 06:48:21.146739 3947 node_ip_handler_linux.go:485] Node address changed to map[10.0.2.36/24:{} 172.20.0.1/32:{}]. Updating annotations. I0429 06:48:21.146985 3947 kube.go:131] Setting annotations map[k8s.ovn.org/host-cidrs:["10.0.2.36/24","172.20.0.1/32"] k8s.ovn.org/l3-gateway-config:{"default":{"mode":"shared","bridge-id":"br-ex","interface-id":"br-ex_ip-10-0-2-36.ec2.internal","mac-address":"0e:da:ef:17:b2:1d","ip-addresses":["10.0.2.36/24"],"ip-address":"10.0.2.36/24","next-hops":["10.0.2.1"],"next-hop":"10.0.2.1","node-port-enable":"true","vlan-id":"0"}} k8s.ovn.org/node-chassis-id:3a75030f-bcc4-42b7-9f55-c26572786127 k8s.ovn.org/node-primary-ifaddr:{"ipv4":"10.0.2.36/24"}] on node ip-10-0-2-36.ec2.internal I0429 06:48:21.165842 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-36.ec2.internal I0429 06:48:21.165865 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-36.ec2.internal: 27.922µs I0429 06:48:21.165915 3947 master.go:548] Adding or Updating local node "ip-10-0-2-36.ec2.internal" for network "default" I0429 06:48:21.165929 3947 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-2-36.ec2.internal in network "default" I0429 06:48:21.165928 3947 egressfirewall.go:723] Syncing node "ip-10-0-2-36.ec2.internal" for egress firewall I0429 06:48:21.165990 3947 node_tracker.go:169] Node ip-10-0-2-36.ec2.internal switch + router changed, syncing services in network "default" I0429 06:48:21.166012 3947 services_controller.go:574] Full service sync requested for network=default I0429 06:48:21.166309 3947 gateway_shared_intf.go:2023] Setting OVN Masquerade route with source: 10.0.2.36 I0429 06:48:21.166426 3947 kube.go:131] Setting annotations map[k8s.ovn.org/node-masquerade-subnet:{"ipv4":"169.254.0.0/17","ipv6":"fd69::/112"}] on node ip-10-0-2-36.ec2.internal I0429 06:48:21.171107 3947 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-2-36.ec2.internal. Updating no re-route policies I0429 06:48:21.171179 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-operator/iptables-alerter-fj6hx I0429 06:48:21.171182 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-node-tuning-operator/tuned-rs7dk I0429 06:48:21.171198 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-operator/iptables-alerter-fj6hx I0429 06:48:21.171200 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-node-tuning-operator/tuned-rs7dk I0429 06:48:21.171200 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-36.ec2.internal I0429 06:48:21.171207 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-s9ckh I0429 06:48:21.171217 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-36.ec2.internal I0429 06:48:21.171219 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-s9ckh I0429 06:48:21.171220 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-node-tuning-operator/tuned-rs7dk after 0 failed attempt(s) I0429 06:48:21.171224 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-operator/iptables-alerter-fj6hx after 0 failed attempt(s) I0429 06:48:21.171226 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-57zbd I0429 06:48:21.171234 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-image-registry/node-ca-j7hjl I0429 06:48:21.171239 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-image-registry/node-ca-j7hjl I0429 06:48:21.171240 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-88f5c I0429 06:48:21.171239 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-57zbd I0429 06:48:21.171240 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-36.ec2.internal after 0 failed attempt(s) I0429 06:48:21.171248 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-88f5c I0429 06:48:21.171249 3947 obj_retry.go:409] Retry add failed for *v1.Pod openshift-multus/network-metrics-daemon-s9ckh, will try again later: failed to obtain IPs to add remote pod openshift-multus/network-metrics-daemon-s9ckh: suppressed error logged: pod openshift-multus/network-metrics-daemon-s9ckh: no pod IPs found I0429 06:48:21.171252 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/network-metrics-daemon-764nj I0429 06:48:21.171261 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-cluster-csi-drivers/aws-ebs-csi-driver-node-57zbd after 0 failed attempt(s) I0429 06:48:21.171264 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-additional-cni-plugins-6mm7d I0429 06:48:21.171266 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-dns/node-resolver-qz9dh I0429 06:48:21.171270 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/network-metrics-daemon-764nj I0429 06:48:21.171267 3947 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/konnectivity-agent-kbnq6 I0429 06:48:21.171277 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-stmtt I0429 06:48:21.171274 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-additional-cni-plugins-6mm7d I0429 06:48:21.171280 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-dns/node-resolver-qz9dh I0429 06:48:21.171283 3947 obj_retry.go:398] Adding new object: *v1.Pod kube-system/konnectivity-agent-kbnq6 I0429 06:48:21.171276 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-network-diagnostics/network-check-target-mkl5n I0429 06:48:21.171286 3947 obj_retry.go:409] Retry add failed for *v1.Pod openshift-network-diagnostics/network-check-target-88f5c, will try again later: failed to obtain IPs to add remote pod openshift-network-diagnostics/network-check-target-88f5c: suppressed error logged: pod openshift-network-diagnostics/network-check-target-88f5c: no pod IPs found I0429 06:48:21.171291 3947 obj_retry.go:334] Retry object setup: *v1.Pod openshift-multus/multus-rh9bb I0429 06:48:21.171273 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-image-registry/node-ca-j7hjl after 0 failed attempt(s) I0429 06:48:21.171296 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-additional-cni-plugins-6mm7d after 0 failed attempt(s) I0429 06:48:21.171303 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-dns/node-resolver-qz9dh after 0 failed attempt(s) I0429 06:48:21.171305 3947 obj_retry.go:418] Retry successful for *v1.Pod kube-system/konnectivity-agent-kbnq6 after 0 failed attempt(s) I0429 06:48:21.171305 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-multus/multus-rh9bb I0429 06:48:21.171304 3947 obj_retry.go:334] Retry object setup: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-36.ec2.internal I0429 06:48:21.171312 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-network-diagnostics/network-check-target-mkl5n I0429 06:48:21.171325 3947 obj_retry.go:398] Adding new object: *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-36.ec2.internal I0429 06:48:21.171327 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/multus-rh9bb after 0 failed attempt(s) I0429 06:48:21.171335 3947 base_network_controller_pods.go:487] [default/openshift-multus/network-metrics-daemon-764nj] creating logical port openshift-multus_network-metrics-daemon-764nj for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:21.171352 3947 base_network_controller_pods.go:487] [default/openshift-network-diagnostics/network-check-target-mkl5n] creating logical port openshift-network-diagnostics_network-check-target-mkl5n for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:21.171352 3947 obj_retry.go:418] Retry successful for *v1.Pod kube-system/kube-apiserver-proxy-ip-10-0-2-36.ec2.internal after 0 failed attempt(s) I0429 06:48:21.171285 3947 obj_retry.go:398] Adding new object: *v1.Pod openshift-ovn-kubernetes/ovnkube-node-stmtt I0429 06:48:21.171424 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-ovn-kubernetes/ovnkube-node-stmtt after 0 failed attempt(s) I0429 06:48:21.171988 3947 pods.go:271] [openshift-network-diagnostics/network-check-target-mkl5n] addLogicalPort took 641.951µs, libovsdb time 265.335µs I0429 06:48:21.172002 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-network-diagnostics/network-check-target-mkl5n after 0 failed attempt(s) I0429 06:48:21.172087 3947 pods.go:271] [openshift-multus/network-metrics-daemon-764nj] addLogicalPort took 760.566µs, libovsdb time 242.627µs I0429 06:48:21.172103 3947 obj_retry.go:418] Retry successful for *v1.Pod openshift-multus/network-metrics-daemon-764nj after 0 failed attempt(s) I0429 06:48:21.183892 3947 gateway_shared_intf.go:1799] Creating Gateway Node Port Watcher I0429 06:48:21.197206 3947 iptables.go:108] Creating table: mangle chain: OUTPUT I0429 06:48:21.200913 3947 iptables.go:108] Creating table: nat chain: OUTPUT I0429 06:48:21.251921 3947 iptables.go:108] Creating table: nat chain: PREROUTING I0429 06:48:21.312759 3947 iptables.go:108] Creating table: filter chain: FORWARD I0429 06:48:21.331793 3947 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.1 W0429 06:48:21.331884 3947 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.1, on iface br-ex: failed to delete neighbour entry 169.254.0.1 : no such file or directory I0429 06:48:21.331928 3947 gateway_shared_intf.go:2124] Ensuring IP Neighbor entry for: 169.254.0.4 W0429 06:48:21.332018 3947 gateway_shared_intf.go:2130] Failed to remove IP neighbor entry for ip 169.254.0.4, on iface br-ex: failed to delete neighbour entry 169.254.0.4 : no such file or directory I0429 06:48:21.332066 3947 gateway_init.go:325] Gateway and management port readiness took 2.726865177s I0429 06:48:21.332094 3947 controller.go:133] Adding controller [node-nad-controller NAD controller] event handlers I0429 06:48:21.332130 3947 shared_informer.go:349] "Waiting for caches to sync" controller="[node-nad-controller NAD controller]" I0429 06:48:21.332144 3947 shared_informer.go:356] "Caches are synced" controller="[node-nad-controller NAD controller]" I0429 06:48:21.332155 3947 nad_controller.go:541] [node-nad-controller NAD controller]: 1 NADs are missing the network ID annotation, fetching from nodes I0429 06:48:21.332211 3947 controller.go:157] Starting controller [node-nad-controller NAD controller] with 1 workers I0429 06:48:21.332294 3947 nad_controller.go:580] [node-nad-controller NAD controller]: finished syncing NAD openshift-ovn-kubernetes/default, took 53.683µs I0429 06:48:21.332549 3947 network_controller.go:373] [node-nad-controller network controller]: syncing all networks I0429 06:48:21.332576 3947 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 16.173µs I0429 06:48:21.332603 3947 network_controller.go:384] [node-nad-controller network controller]: finished syncing all networks. Time taken: 53.75µs I0429 06:48:21.332610 3947 controller.go:157] Starting controller [node-nad-controller network controller] with 1 workers I0429 06:48:21.332619 3947 nad_controller.go:379] [node-nad-controller NAD controller]: started I0429 06:48:21.332626 3947 default_node_network_controller.go:899] Starting the default node network controller I0429 06:48:21.332631 3947 gateway_init.go:335] Initializing Gateway Functionality for gateway Start I0429 06:48:21.332650 3947 network_controller.go:402] [node-nad-controller network controller]: finished syncing network default, took 17.611µs I0429 06:48:21.332818 3947 default_node_network_controller.go:1460] MTU (9001) of network interface br-ex is big enough to deal with Geneve header overhead (sum 8959). I0429 06:48:21.332827 3947 default_node_network_controller.go:935] Waiting for gateway and management port readiness... I0429 06:48:21.332865 3947 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc00097e640 0xc00027fea0 0xc00027ff10 0xc0002f8000 0xc0002f8070 0xc0002f8150 0xc0002f81c0 0xc0001374f0 0xc0002f82a0 0xc0002f8310 0xc0001243c0 0xc00641d5e0 0xc0002f8380 map[0x2ff9080:0xc002195320 0x2ff95c0:0xc0021954a0 0x2ffa2e0:0xc002195260 0x2ffa580:0xc0021952c0 0x2ffad60:0xc002195380 0x2ffb000:0xc0021953e0 0x2ffb2a0:0xc002195440 0x2ffba80:0xc002195500 0x2ffbd20:0xc002195920 0x3026980:0xc0010c2960 0x3026d20:0xc0010c29c0 0x3027800:0xc000c0b5c0 0x3029160:0xc001bd6000 0x30298a0:0xc0010c2ba0 0x302a720:0xc001bd6060] 0xc0009462a0 0} I0429 06:48:21.332922 3947 gateway.go:151] Starting gateway service sync I0429 06:48:21.333354 3947 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ITP I0429 06:48:21.335902 3947 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-NODEPORT I0429 06:48:21.366529 3947 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-EXTERNALIP I0429 06:48:21.367964 3947 gateway_iptables.go:474] Recreating iptables rules for table: nat, chain: OVN-KUBE-ETP I0429 06:48:21.369341 3947 gateway_iptables.go:474] Recreating iptables rules for table: mangle, chain: OVN-KUBE-ITP I0429 06:48:21.406284 3947 gateway.go:168] Gateway service sync done. Time taken: 73.346623ms I0429 06:48:21.406848 3947 healthcheck.go:145] Opening healthcheck "openshift-ingress/router-default" on port 30173 I0429 06:48:21.407164 3947 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc00097e640 0xc00027fea0 0xc00027ff10 0xc0002f8000 0xc0002f8070 0xc0002f8150 0xc0002f81c0 0xc0001374f0 0xc0002f82a0 0xc0002f8310 0xc0001243c0 0xc00641d5e0 0xc0002f8380 map[0x2ff9080:0xc002195320 0x2ff95c0:0xc0021954a0 0x2ffa2e0:0xc002195260 0x2ffa580:0xc0021952c0 0x2ffad60:0xc002195380 0x2ffb000:0xc0021953e0 0x2ffb2a0:0xc002195440 0x2ffba80:0xc002195500 0x2ffbd20:0xc002195920 0x3026980:0xc0010c2960 0x3026d20:0xc0010c29c0 0x3027800:0xc000c0b5c0 0x3029160:0xc001bd6000 0x30298a0:0xc0010c2ba0 0x302a720:0xc001bd6060] 0xc0009462a0 0} I0429 06:48:21.407727 3947 obj_retry_gateway.go:33] [newRetryFrameworkNodeWithParameters] g.watchFactory=&{0xc00097e640 0xc00027fea0 0xc00027ff10 0xc0002f8000 0xc0002f8070 0xc0002f8150 0xc0002f81c0 0xc0001374f0 0xc0002f82a0 0xc0002f8310 0xc0001243c0 0xc00641d5e0 0xc0002f8380 map[0x2ff9080:0xc002195320 0x2ff95c0:0xc0021954a0 0x2ffa2e0:0xc002195260 0x2ffa580:0xc0021952c0 0x2ffad60:0xc002195380 0x2ffb000:0xc0021953e0 0x2ffb2a0:0xc002195440 0x2ffba80:0xc002195500 0x2ffbd20:0xc002195920 0x3026980:0xc0010c2960 0x3026d20:0xc0010c29c0 0x3027800:0xc000c0b5c0 0x3029160:0xc001bd6000 0x30298a0:0xc0010c2ba0 0x302a720:0xc001bd6060] 0xc0009462a0 0} I0429 06:48:21.407765 3947 gateway.go:532] Reconciling gateway with updates I0429 06:48:21.413496 3947 gateway.go:339] Spawning Conntrack Rule Check Thread I0429 06:48:21.413580 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 06:48:21.413609 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 06:48:21.413619 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 06:48:21.413626 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 06:48:21.413623 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0429 06:48:21.413637 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 06:48:21.413644 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0429 06:48:21.413651 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 06:48:21.413652 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 06:48:21.413669 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 06:48:21.413681 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 06:48:21.413687 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 06:48:21.413688 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0429 06:48:21.413687 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 06:48:21.413695 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0429 06:48:21.413698 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0429 06:48:21.413701 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0429 06:48:21.413704 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0429 06:48:21.413702 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 06:48:21.413698 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 06:48:21.413708 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 06:48:21.413713 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 06:48:21.413718 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 06:48:21.413725 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 06:48:21.413729 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 06:48:21.413729 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 06:48:21.413726 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 06:48:21.413744 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0429 06:48:21.413748 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 06:48:21.413765 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 06:48:21.413779 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0429 06:48:21.413675 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0429 06:48:21.413711 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 06:48:21.413792 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0429 06:48:21.413797 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0429 06:48:21.413708 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0429 06:48:21.413803 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 06:48:21.413813 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0429 06:48:21.413812 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0429 06:48:21.413812 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 06:48:21.413834 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 06:48:21.413840 3947 default_node_network_controller.go:944] Gateway and management port readiness took 81.005294ms I0429 06:48:21.413866 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0429 06:48:21.413818 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0429 06:48:21.413881 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0429 06:48:21.413907 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0429 06:48:21.413919 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0429 06:48:21.413928 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0429 06:48:21.413935 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0429 06:48:21.413938 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0429 06:48:21.413945 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0429 06:48:21.413960 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 06:48:21.413968 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 06:48:21.413974 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 06:48:21.413978 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 06:48:21.413978 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 06:48:21.413990 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 06:48:21.413994 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 06:48:21.413995 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 06:48:21.414001 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 06:48:21.413994 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 06:48:21.414000 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 06:48:21.414011 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 06:48:21.414012 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 06:48:21.414021 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0429 06:48:21.413960 3947 port_claim.go:71] Svc openshift-ingress/router-default: port 30928 is already open, no action needed I0429 06:48:21.413969 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0429 06:48:21.414042 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0429 06:48:21.414040 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0429 06:48:21.414047 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 06:48:21.414053 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0429 06:48:21.414061 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 06:48:21.414061 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0429 06:48:21.414067 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0429 06:48:21.414071 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 06:48:21.413982 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 06:48:21.414083 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 06:48:21.414091 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0429 06:48:21.414100 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0429 06:48:21.414065 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0429 06:48:21.414104 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0429 06:48:21.413963 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0429 06:48:21.414124 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0429 06:48:21.414150 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0429 06:48:21.414169 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0429 06:48:21.414157 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0429 06:48:21.414266 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0429 06:48:21.413982 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 06:48:21.414282 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0429 06:48:21.414289 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 06:48:21.414043 3947 port_claim.go:71] Svc openshift-ingress/router-default: port 31311 is already open, no action needed I0429 06:48:21.414055 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0429 06:48:21.414058 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0429 06:48:21.414395 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0429 06:48:21.414402 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0429 06:48:21.413971 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 06:48:21.414412 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 06:48:21.414417 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0429 06:48:21.414212 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 06:48:21.414426 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 06:48:21.414237 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 06:48:21.414442 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 06:48:21.414236 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0429 06:48:21.414455 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0429 06:48:21.414463 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0429 06:48:21.414237 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0429 06:48:21.414514 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0429 06:48:21.414247 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 06:48:21.414600 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 06:48:21.414242 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0429 06:48:21.414616 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0429 06:48:21.414243 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 06:48:21.414673 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 06:48:21.414248 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 06:48:21.414709 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 06:48:21.414716 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0429 06:48:21.414257 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 06:48:21.414735 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 06:48:21.414256 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 06:48:21.414773 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 06:48:21.414261 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 06:48:21.414815 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 06:48:21.415203 3947 node_ip_handler_linux.go:160] Node IP manager is running W0429 06:48:21.454520 3947 port_linux.go:245] Missing or unable to find route entry for subnet 10.128.0.0/14 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0429 06:48:21.454761 3947 port_linux.go:245] Missing or unable to find route entry for subnet 169.254.0.3/32 via gateway 10.129.0.1 on link ovn-k8s-mp0 with MTU: 0 W0429 06:48:21.454888 3947 port_linux.go:261] Missing arp entry for MAC/IP binding (0a:58:0a:81:00:01/10.129.0.1) on link ovn-k8s-mp0 I0429 06:48:21.464776 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0429 06:48:21.464808 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0429 06:48:21.509723 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0429 06:48:21.535934 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0429 06:48:21.535948 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0429 06:48:21.535967 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0429 06:48:21.535973 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0429 06:48:21.536003 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0429 06:48:21.536010 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0429 06:48:21.536016 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0429 06:48:21.536042 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0429 06:48:21.536051 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0429 06:48:21.536056 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0429 06:48:22.633680 3947 default_network_controller.go:1025] Node "ip-10-0-0-41.ec2.internal" in remote zone "global", network "default", needs interconnect zone sync up. Zone cluster changed: false W0429 06:48:23.389187 3947 egressip_healthcheck.go:76] Health checking using insecure connection I0429 06:48:23.389236 3947 default_node_network_controller.go:1380] Starting node controller node sync I0429 06:48:23.389241 3947 egressip_healthcheck.go:109] Starting Egress IP Health Server on 10.129.0.2:9107 I0429 06:48:23.401193 3947 default_node_network_controller.go:1422] Node controller node sync done. Time taken: 11.943492ms I0429 06:48:23.401304 3947 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-61.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0429 06:48:23.401319 3947 default_node_network_controller.go:1293] Adding remote node "ip-10-0-1-61.ec2.internal", IP: 10.0.1.61 to PMTUD blocking rules I0429 06:48:23.407196 3947 healthcheck_node.go:124] "Starting node proxy healthz server" address="0.0.0.0:10256" I0429 06:48:23.407465 3947 egressservice_node.go:93] Setting up event handlers for Egress Services I0429 06:48:23.407576 3947 egressservice_node.go:183] Starting Egress Services Controller I0429 06:48:23.407609 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices" I0429 06:48:23.407621 3947 shared_informer.go:356] "Caches are synced" controller="egressservices" I0429 06:48:23.407627 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_services" I0429 06:48:23.407634 3947 shared_informer.go:356] "Caches are synced" controller="egressservices_services" I0429 06:48:23.407640 3947 shared_informer.go:349] "Waiting for caches to sync" controller="egressservices_endpointslices" I0429 06:48:23.407645 3947 shared_informer.go:356] "Caches are synced" controller="egressservices_endpointslices" I0429 06:48:23.407649 3947 egressservice_node.go:197] Repairing Egress Services I0429 06:48:23.418058 3947 node_controller.go:43] Starting Admin Policy Based Route Node Controller I0429 06:48:23.418070 3947 external_controller.go:278] Starting Admin Policy Based Route Controller I0429 06:48:23.418357 3947 default_node_network_controller.go:1077] Egress IP for secondary host network is disabled I0429 06:48:23.418373 3947 link_network_manager.go:119] Link manager is running I0429 06:48:23.418388 3947 default_node_network_controller.go:1098] Default node network controller initialized and ready. I0429 06:48:23.418516 3947 ovspinning_linux.go:50] OVS CPU affinity pinning disabled I0429 06:48:23.418833 3947 vrf_manager.go:128] VRF manager is running I0429 06:48:23.419150 3947 node_controller_manager.go:437] Removing flows to drop GARP I0429 06:48:23.419159 3947 gateway.go:532] Reconciling gateway with updates I0429 06:48:23.439487 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 06:48:23.439508 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 06:48:23.439511 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/packageserver I0429 06:48:23.439516 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 06:48:23.439520 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-canary/ingress-canary I0429 06:48:23.439526 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/packageserver I0429 06:48:23.439530 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 06:48:23.439530 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 06:48:23.439541 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-samples-operator/metrics I0429 06:48:23.439542 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 06:48:23.439552 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-samples-operator/metrics after 0 failed attempt(s) I0429 06:48:23.439557 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns/dns-default I0429 06:48:23.439570 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-operator/metrics I0429 06:48:23.439574 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns/dns-default I0429 06:48:23.439575 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-default I0429 06:48:23.439578 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-operator/metrics I0429 06:48:23.439605 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-default I0429 06:48:23.439606 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 06:48:23.439608 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 06:48:23.439614 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-console-operator/metrics I0429 06:48:23.439580 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 06:48:23.439623 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 06:48:23.439625 3947 port_claim.go:71] Svc openshift-ingress/router-default: port 30928 is already open, no action needed I0429 06:48:23.439630 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress-operator/metrics I0429 06:48:23.439632 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ingress/router-internal-default I0429 06:48:23.439631 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 06:48:23.439638 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 06:48:23.439638 3947 port_claim.go:71] Svc openshift-ingress/router-default: port 31311 is already open, no action needed I0429 06:48:23.439650 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-server I0429 06:48:23.439651 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator I0429 06:48:23.439524 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-config-operator/metrics I0429 06:48:23.439657 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift I0429 06:48:23.439679 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 06:48:23.439678 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 06:48:23.439514 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-authentication-operator/metrics I0429 06:48:23.439697 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics I0429 06:48:23.439494 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-insights/metrics I0429 06:48:23.439705 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service I0429 06:48:23.439714 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 06:48:23.439723 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics I0429 06:48:23.439724 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 06:48:23.439731 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 06:48:23.439736 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-insights/metrics I0429 06:48:23.439608 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-operator/metrics after 0 failed attempt(s) I0429 06:48:23.439563 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics I0429 06:48:23.439753 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-image-registry/image-registry I0429 06:48:23.439625 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-console-operator/metrics I0429 06:48:23.439768 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 06:48:23.439774 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 06:48:23.439615 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 06:48:23.439782 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 06:48:23.439787 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon I0429 06:48:23.439785 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 06:48:23.439792 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog I0429 06:48:23.439796 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 06:48:23.439803 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver I0429 06:48:23.439812 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics I0429 06:48:23.439812 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 06:48:23.439816 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-machine-approver/machine-approver after 0 failed attempt(s) I0429 06:48:23.439821 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-source I0429 06:48:23.439828 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver/check-endpoints I0429 06:48:23.439832 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-source after 0 failed attempt(s) I0429 06:48:23.439828 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 06:48:23.439846 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-apiserver-operator/metrics I0429 06:48:23.439769 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 06:48:23.439845 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 06:48:23.439753 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 06:48:23.439858 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics I0429 06:48:23.439862 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-service-ca-operator/metrics I0429 06:48:23.439865 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller I0429 06:48:23.439488 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 06:48:23.439511 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 06:48:23.439908 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-console/networking-console-plugin I0429 06:48:23.439616 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-metrics after 0 failed attempt(s) I0429 06:48:23.439618 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio I0429 06:48:23.439622 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator I0429 06:48:23.440037 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/node-tuning-operator after 0 failed attempt(s) I0429 06:48:23.439666 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-config-operator/metrics I0429 06:48:23.440064 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/kubernetes I0429 06:48:23.439663 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-monitoring/cluster-monitoring-operator after 0 failed attempt(s) I0429 06:48:23.440073 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway default/openshift-apiserver I0429 06:48:23.439671 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 06:48:23.440078 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/kubernetes I0429 06:48:23.440088 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node I0429 06:48:23.440093 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ovn-kubernetes/ovn-kubernetes-node after 0 failed attempt(s) I0429 06:48:23.440088 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-apiserver I0429 06:48:23.439690 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift I0429 06:48:23.440118 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift after 0 failed attempt(s) I0429 06:48:23.439737 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-network-diagnostics/network-check-target I0429 06:48:23.439741 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-version/cluster-version-operator I0429 06:48:23.439749 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 06:48:23.440174 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics I0429 06:48:23.439763 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-image-registry/image-registry I0429 06:48:23.439782 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway default/openshift-oauth-apiserver I0429 06:48:23.439897 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-multus/network-metrics-service I0429 06:48:23.440370 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-multus/network-metrics-service after 0 failed attempt(s) I0429 06:48:23.439580 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 06:48:23.440381 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics I0429 06:48:23.440478 3947 obj_retry.go:334] Retry object setup: *factory.serviceForGateway openshift-dns-operator/metrics I0429 06:48:23.440492 3947 obj_retry.go:398] Adding new object: *factory.serviceForGateway openshift-dns-operator/metrics I0429 06:48:23.460267 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns/dns-default after 0 failed attempt(s) I0429 06:48:23.460337 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-canary/ingress-canary after 0 failed attempt(s) I0429 06:48:23.460358 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-internal-default after 0 failed attempt(s) I0429 06:48:23.460365 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-server after 0 failed attempt(s) I0429 06:48:23.500231 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress/router-default after 0 failed attempt(s) I0429 06:48:23.500243 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-ingress-operator/metrics after 0 failed attempt(s) I0429 06:48:23.500262 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/packageserver after 0 failed attempt(s) I0429 06:48:23.500273 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-authentication-operator/metrics after 0 failed attempt(s) I0429 06:48:23.500279 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/cluster-storage-operator-metrics after 0 failed attempt(s) I0429 06:48:23.500285 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-storage-version-migrator-operator/metrics after 0 failed attempt(s) I0429 06:48:23.500291 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-node-tuning-operator/performance-addon-operator-service after 0 failed attempt(s) I0429 06:48:23.500298 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-kube-controller-manager-operator/metrics after 0 failed attempt(s) I0429 06:48:23.500306 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-insights/metrics after 0 failed attempt(s) I0429 06:48:23.500314 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-daemon after 0 failed attempt(s) I0429 06:48:23.500320 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cloud-credential-operator/cco-metrics after 0 failed attempt(s) I0429 06:48:23.500325 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver/check-endpoints after 0 failed attempt(s) I0429 06:48:23.500334 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-apiserver-operator/metrics after 0 failed attempt(s) I0429 06:48:23.500340 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/catalog-operator-metrics after 0 failed attempt(s) I0429 06:48:23.500345 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-service-ca-operator/metrics after 0 failed attempt(s) I0429 06:48:23.500351 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/machine-config-controller after 0 failed attempt(s) I0429 06:48:23.500357 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-deployment-validation-operator/deployment-validation-operator-catalog after 0 failed attempt(s) I0429 06:48:23.500371 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-console-operator/metrics after 0 failed attempt(s) I0429 06:48:23.500384 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-machine-config-operator/kube-rbac-proxy-crio after 0 failed attempt(s) I0429 06:48:23.500399 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-config-operator/metrics after 0 failed attempt(s) I0429 06:48:23.500405 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-console/networking-console-plugin after 0 failed attempt(s) I0429 06:48:23.545556 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/kubernetes after 0 failed attempt(s) I0429 06:48:23.545596 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-network-diagnostics/network-check-target after 0 failed attempt(s) I0429 06:48:23.545604 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-apiserver after 0 failed attempt(s) I0429 06:48:23.545610 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-version/cluster-version-operator after 0 failed attempt(s) I0429 06:48:23.545615 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-cluster-storage-operator/csi-snapshot-controller-operator-metrics after 0 failed attempt(s) I0429 06:48:23.545625 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-operator-lifecycle-manager/olm-operator-metrics after 0 failed attempt(s) I0429 06:48:23.545642 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-image-registry/image-registry after 0 failed attempt(s) I0429 06:48:23.545648 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway openshift-dns-operator/metrics after 0 failed attempt(s) I0429 06:48:23.545653 3947 obj_retry.go:418] Retry successful for *factory.serviceForGateway default/openshift-oauth-apiserver after 0 failed attempt(s) I0429 06:48:23.619815 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-1-61.ec2.internal I0429 06:48:23.619842 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-1-61.ec2.internal: 34.171µs I0429 06:48:25.411518 3947 default_network_controller.go:1025] Node "ip-10-0-0-41.ec2.internal" in remote zone "ip-10-0-0-41.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 06:48:25.411575 3947 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-41.ec2.internal in network "default" I0429 06:48:25.411611 3947 node_tracker.go:235] Node ip-10-0-0-41.ec2.internal has invalid / no gateway config: k8s.ovn.org/l3-gateway-config annotation not found for node "ip-10-0-0-41.ec2.internal" I0429 06:48:25.411625 3947 node_tracker.go:169] Node ip-10-0-0-41.ec2.internal switch + router changed, syncing services in network "default" I0429 06:48:25.411640 3947 services_controller.go:574] Full service sync requested for network=default I0429 06:48:25.446836 3947 default_network_controller.go:1025] Node "ip-10-0-0-41.ec2.internal" in remote zone "ip-10-0-0-41.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 06:48:25.559073 3947 default_network_controller.go:1025] Node "ip-10-0-0-41.ec2.internal" in remote zone "ip-10-0-0-41.ec2.internal", network "default", needs interconnect zone sync up. Zone cluster changed: false I0429 06:48:25.559120 3947 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-41.ec2.internal in network "default" I0429 06:48:25.559179 3947 node_tracker.go:169] Node ip-10-0-0-41.ec2.internal switch + router changed, syncing services in network "default" I0429 06:48:25.559198 3947 services_controller.go:574] Full service sync requested for network=default I0429 06:48:25.560121 3947 zone_ic_handler.go:195] Time taken to create transit switch: 252.693µs I0429 06:48:25.560136 3947 zone_ic_handler.go:277] Creating interconnect resources for remote zone node ip-10-0-0-41.ec2.internal for the network default I0429 06:48:25.561124 3947 zone_ic_handler.go:282] Creating Interconnect resources for node "ip-10-0-0-41.ec2.internal" on network "default" took: 1.266816ms I0429 06:48:25.703921 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-2-36.ec2.internal I0429 06:48:25.703950 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-2-36.ec2.internal: 35.567µs I0429 06:48:25.739648 3947 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-76d6dbccdd-52h2m] creating logical port openshift-image-registry_image-registry-76d6dbccdd-52h2m for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:25.739737 3947 kube.go:256] Updating pod openshift-image-registry/image-registry-76d6dbccdd-52h2m I0429 06:48:25.739740 3947 base_network_controller_pods.go:487] [default/openshift-ingress/router-default-7d9696c95d-4vdv8] creating logical port openshift-ingress_router-default-7d9696c95d-4vdv8 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:25.739812 3947 kube.go:256] Updating pod openshift-ingress/router-default-7d9696c95d-4vdv8 I0429 06:48:25.760792 3947 pod.go:62] [openshift-ingress/router-default-7d9696c95d-4vdv8] pod update took 20.998788ms I0429 06:48:25.760815 3947 base_network_controller_pods.go:951] [default/openshift-ingress/router-default-7d9696c95d-4vdv8] addLogicalPort annotation time took 21.025385ms I0429 06:48:25.761911 3947 pods.go:271] [openshift-ingress/router-default-7d9696c95d-4vdv8] addLogicalPort took 22.181151ms, libovsdb time 825.155µs I0429 06:48:25.768559 3947 pod.go:62] [openshift-image-registry/image-registry-76d6dbccdd-52h2m] pod update took 28.849237ms I0429 06:48:25.768598 3947 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-76d6dbccdd-52h2m] addLogicalPort annotation time took 28.873783ms I0429 06:48:25.769608 3947 pods.go:271] [openshift-image-registry/image-registry-76d6dbccdd-52h2m] addLogicalPort took 29.974239ms, libovsdb time 732.848µs I0429 06:48:25.781940 3947 base_network_controller_pods.go:487] [default/openshift-dns/dns-default-29lmq] creating logical port openshift-dns_dns-default-29lmq for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:25.782034 3947 kube.go:256] Updating pod openshift-dns/dns-default-29lmq I0429 06:48:25.784647 3947 base_network_controller_pods.go:487] [default/openshift-ingress-canary/ingress-canary-9hd9r] creating logical port openshift-ingress-canary_ingress-canary-9hd9r for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:25.784728 3947 kube.go:256] Updating pod openshift-ingress-canary/ingress-canary-9hd9r I0429 06:48:25.804937 3947 pod.go:62] [openshift-ingress-canary/ingress-canary-9hd9r] pod update took 20.226116ms I0429 06:48:25.804964 3947 base_network_controller_pods.go:951] [default/openshift-ingress-canary/ingress-canary-9hd9r] addLogicalPort annotation time took 20.26038ms I0429 06:48:25.805071 3947 pod.go:62] [openshift-dns/dns-default-29lmq] pod update took 23.075307ms I0429 06:48:25.805092 3947 base_network_controller_pods.go:951] [default/openshift-dns/dns-default-29lmq] addLogicalPort annotation time took 23.097ms I0429 06:48:25.806175 3947 pods.go:271] [openshift-ingress-canary/ingress-canary-9hd9r] addLogicalPort took 21.541419ms, libovsdb time 869.676µs I0429 06:48:25.806394 3947 pods.go:271] [openshift-dns/dns-default-29lmq] addLogicalPort took 24.472138ms, libovsdb time 910.813µs I0429 06:48:26.505738 3947 cni.go:452] [openshift-ingress-canary/ingress-canary-9hd9r 1ce37724cd333d606aecc29df3abfcfa8fa7a03dc7c146aefbf655752c89d40e network default NAD default NAD key ] ADD starting CNI request [openshift-ingress-canary/ingress-canary-9hd9r 1ce37724cd333d606aecc29df3abfcfa8fa7a03dc7c146aefbf655752c89d40e network default NAD default NAD key ] I0429 06:48:26.547054 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress-canary, podName: ingress-canary-9hd9r, hostIfaceName: 1ce37724cd333d6, network: default, NAD default, SandboxID: "1ce37724cd333d606aecc29df3abfcfa8fa7a03dc7c146aefbf655752c89d40e", PCI device ID: , UID: "41f2d37f-237d-4b07-b158-81b29424c5c9", MAC: 0a:58:0a:81:00:08, IPs: [10.129.0.8/23] I0429 06:48:26.602366 3947 cni.go:473] [openshift-ingress-canary/ingress-canary-9hd9r 1ce37724cd333d606aecc29df3abfcfa8fa7a03dc7c146aefbf655752c89d40e network default NAD default NAD key default] ADD finished CNI request [openshift-ingress-canary/ingress-canary-9hd9r 1ce37724cd333d606aecc29df3abfcfa8fa7a03dc7c146aefbf655752c89d40e network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"d6:6b:fd:22:ee:cb\",\"name\":\"1ce37724cd333d6\"},{\"mac\":\"0a:58:0a:81:00:08\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/5bd3d011-9727-4372-b8ae-126b973b4f9f\"}],\"ips\":[{\"address\":\"10.129.0.8/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:48:27.618282 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-41.ec2.internal I0429 06:48:27.618308 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-41.ec2.internal: 32.502µs I0429 06:48:27.618310 3947 egressfirewall.go:723] Syncing node "ip-10-0-0-41.ec2.internal" for egress firewall I0429 06:48:27.618312 3947 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-41.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0429 06:48:27.618310 3947 node_tracker.go:208] Processing possible switch / router updates for node ip-10-0-0-41.ec2.internal in network "default" I0429 06:48:27.618329 3947 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-41.ec2.internal", IP: 10.0.0.41 to PMTUD blocking rules I0429 06:48:27.618371 3947 node_tracker.go:169] Node ip-10-0-0-41.ec2.internal switch + router changed, syncing services in network "default" I0429 06:48:27.618384 3947 services_controller.go:574] Full service sync requested for network=default I0429 06:48:27.624645 3947 default_network_controller.go:1084] Egress IP detected IP address change for node ip-10-0-0-41.ec2.internal. Updating no re-route policies I0429 06:48:27.633496 3947 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-41.ec2.internal", IP: 10.0.0.41 to PMTUD blocking rules I0429 06:48:27.633517 3947 default_node_network_controller.go:1293] Adding remote node "ip-10-0-0-41.ec2.internal", IP: 172.20.0.1 to PMTUD blocking rules I0429 06:48:31.648941 3947 egressservice_zone_node.go:111] Processing sync for Egress Service node ip-10-0-0-41.ec2.internal I0429 06:48:31.648978 3947 egressservice_zone_node.go:114] Finished syncing Egress Service node ip-10-0-0-41.ec2.internal: 41.463µs W0429 06:48:34.616911 3947 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-catalog W0429 06:48:34.645002 3947 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 06:48:34.735089 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0429 06:48:35.605785 3947 namespace.go:98] [openshift-kube-storage-version-migrator] adding namespace I0429 06:48:35.606752 3947 namespace.go:102] [openshift-kube-storage-version-migrator] adding namespace took 942.276µs I0429 06:48:35.636843 3947 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace I0429 06:48:35.813222 3947 namespace.go:98] [openshift-service-ca] adding namespace I0429 06:48:35.814154 3947 namespace.go:102] [openshift-service-ca] adding namespace took 908.906µs I0429 06:48:35.833870 3947 namespace.go:142] [openshift-service-ca] updating namespace W0429 06:48:39.465734 3947 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 06:48:45.888059 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console-operator/metrics I0429 06:48:55.315324 3947 cni.go:452] [openshift-multus/network-metrics-daemon-764nj d4bf163f25d7413dd828e3b6e3a770af507624ba82334b8c44a36a2b9e5a7aa4 network default NAD default NAD key ] ADD starting CNI request [openshift-multus/network-metrics-daemon-764nj d4bf163f25d7413dd828e3b6e3a770af507624ba82334b8c44a36a2b9e5a7aa4 network default NAD default NAD key ] I0429 06:48:55.325279 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-multus, podName: network-metrics-daemon-764nj, hostIfaceName: d4bf163f25d7413, network: default, NAD default, SandboxID: "d4bf163f25d7413dd828e3b6e3a770af507624ba82334b8c44a36a2b9e5a7aa4", PCI device ID: , UID: "3a7731ed-b276-4ded-aeb8-c58bfbaebc18", MAC: 0a:58:0a:81:00:03, IPs: [10.129.0.3/23] I0429 06:48:55.379771 3947 cni.go:473] [openshift-multus/network-metrics-daemon-764nj d4bf163f25d7413dd828e3b6e3a770af507624ba82334b8c44a36a2b9e5a7aa4 network default NAD default NAD key default] ADD finished CNI request [openshift-multus/network-metrics-daemon-764nj d4bf163f25d7413dd828e3b6e3a770af507624ba82334b8c44a36a2b9e5a7aa4 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:8d:b4:04:38:25\",\"name\":\"d4bf163f25d7413\"},{\"mac\":\"0a:58:0a:81:00:03\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/60f281a6-eb07-4f51-bc91-a7848ea877f4\"}],\"ips\":[{\"address\":\"10.129.0.3/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:48:55.600940 3947 cni.go:452] [openshift-network-diagnostics/network-check-target-mkl5n 4a00f041a97105f2279ce52e66e5d05632b2e6e80a7e7e54c0ac14bb24c5ed22 network default NAD default NAD key ] ADD starting CNI request [openshift-network-diagnostics/network-check-target-mkl5n 4a00f041a97105f2279ce52e66e5d05632b2e6e80a7e7e54c0ac14bb24c5ed22 network default NAD default NAD key ] I0429 06:48:55.611182 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-network-diagnostics, podName: network-check-target-mkl5n, hostIfaceName: 4a00f041a97105f, network: default, NAD default, SandboxID: "4a00f041a97105f2279ce52e66e5d05632b2e6e80a7e7e54c0ac14bb24c5ed22", PCI device ID: , UID: "b832700f-bcf4-4ef1-a529-c28de07285b1", MAC: 0a:58:0a:81:00:04, IPs: [10.129.0.4/23] I0429 06:48:55.657066 3947 cni.go:473] [openshift-network-diagnostics/network-check-target-mkl5n 4a00f041a97105f2279ce52e66e5d05632b2e6e80a7e7e54c0ac14bb24c5ed22 network default NAD default NAD key default] ADD finished CNI request [openshift-network-diagnostics/network-check-target-mkl5n 4a00f041a97105f2279ce52e66e5d05632b2e6e80a7e7e54c0ac14bb24c5ed22 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:15:18:4a:89:8a\",\"name\":\"4a00f041a97105f\"},{\"mac\":\"0a:58:0a:81:00:04\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8e644bac-4428-4742-aebc-e181f70fc8da\"}],\"ips\":[{\"address\":\"10.129.0.4/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:48:58.360873 3947 cni.go:452] [openshift-image-registry/image-registry-76d6dbccdd-52h2m 6ce1d7cfff8106f1d69f306defdf81c1349ecd4b423c1bd12f1e3b8741d20f59 network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-76d6dbccdd-52h2m 6ce1d7cfff8106f1d69f306defdf81c1349ecd4b423c1bd12f1e3b8741d20f59 network default NAD default NAD key ] I0429 06:48:58.365326 3947 cni.go:452] [openshift-dns/dns-default-29lmq 3e18887c668bb63b230f312ff73d4d84c355583fdf97dd97e4b16de1eb994a9f network default NAD default NAD key ] ADD starting CNI request [openshift-dns/dns-default-29lmq 3e18887c668bb63b230f312ff73d4d84c355583fdf97dd97e4b16de1eb994a9f network default NAD default NAD key ] I0429 06:48:58.366133 3947 cni.go:452] [openshift-ingress/router-default-7d9696c95d-4vdv8 efb80d67914b7efafbaefb93480a34816a618897854d365facc327ab801622a6 network default NAD default NAD key ] ADD starting CNI request [openshift-ingress/router-default-7d9696c95d-4vdv8 efb80d67914b7efafbaefb93480a34816a618897854d365facc327ab801622a6 network default NAD default NAD key ] I0429 06:48:58.371843 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-76d6dbccdd-52h2m, hostIfaceName: 6ce1d7cfff8106f, network: default, NAD default, SandboxID: "6ce1d7cfff8106f1d69f306defdf81c1349ecd4b423c1bd12f1e3b8741d20f59", PCI device ID: , UID: "a17543bf-82ec-419d-9577-74cfe678ee8f", MAC: 0a:58:0a:81:00:05, IPs: [10.129.0.5/23] I0429 06:48:58.379527 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-dns, podName: dns-default-29lmq, hostIfaceName: 3e18887c668bb63, network: default, NAD default, SandboxID: "3e18887c668bb63b230f312ff73d4d84c355583fdf97dd97e4b16de1eb994a9f", PCI device ID: , UID: "19295b4c-8542-402c-b9af-7859ea15602b", MAC: 0a:58:0a:81:00:07, IPs: [10.129.0.7/23] I0429 06:48:58.380711 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-ingress, podName: router-default-7d9696c95d-4vdv8, hostIfaceName: efb80d67914b7ef, network: default, NAD default, SandboxID: "efb80d67914b7efafbaefb93480a34816a618897854d365facc327ab801622a6", PCI device ID: , UID: "0ba95545-1f39-41be-99ea-edde2ec1bb66", MAC: 0a:58:0a:81:00:06, IPs: [10.129.0.6/23] I0429 06:48:58.422109 3947 cni.go:473] [openshift-image-registry/image-registry-76d6dbccdd-52h2m 6ce1d7cfff8106f1d69f306defdf81c1349ecd4b423c1bd12f1e3b8741d20f59 network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-76d6dbccdd-52h2m 6ce1d7cfff8106f1d69f306defdf81c1349ecd4b423c1bd12f1e3b8741d20f59 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"76:fe:b0:56:56:df\",\"name\":\"6ce1d7cfff8106f\"},{\"mac\":\"0a:58:0a:81:00:05\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/079d9d30-7e39-4f34-b8f8-5208064eb477\"}],\"ips\":[{\"address\":\"10.129.0.5/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:48:58.645317 3947 cni.go:473] [openshift-dns/dns-default-29lmq 3e18887c668bb63b230f312ff73d4d84c355583fdf97dd97e4b16de1eb994a9f network default NAD default NAD key default] ADD finished CNI request [openshift-dns/dns-default-29lmq 3e18887c668bb63b230f312ff73d4d84c355583fdf97dd97e4b16de1eb994a9f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:e7:73:35:a5:49\",\"name\":\"3e18887c668bb63\"},{\"mac\":\"0a:58:0a:81:00:07\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f2ab421b-efd6-4d3e-8c01-2e2e3aac0c44\"}],\"ips\":[{\"address\":\"10.129.0.7/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:48:58.645360 3947 cni.go:473] [openshift-ingress/router-default-7d9696c95d-4vdv8 efb80d67914b7efafbaefb93480a34816a618897854d365facc327ab801622a6 network default NAD default NAD key default] ADD finished CNI request [openshift-ingress/router-default-7d9696c95d-4vdv8 efb80d67914b7efafbaefb93480a34816a618897854d365facc327ab801622a6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"02:cc:2e:bc:1c:31\",\"name\":\"efb80d67914b7ef\"},{\"mac\":\"0a:58:0a:81:00:06\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3055ac9a-8695-402b-afe1-f35b0edacd3e\"}],\"ips\":[{\"address\":\"10.129.0.6/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:48:58.840699 3947 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry W0429 06:48:58.843501 3947 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 06:48:58.846037 3947 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0429 06:48:58.846055 3947 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0429 06:48:58.846060 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0429 06:48:58.895486 3947 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0429 06:48:58.911851 3947 base_network_controller_pods.go:487] [default/openshift-console/downloads-5c54f4944f-ft449] creating logical port openshift-console_downloads-5c54f4944f-ft449 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:58.911943 3947 kube.go:256] Updating pod openshift-console/downloads-5c54f4944f-ft449 W0429 06:48:58.925555 3947 util.go:870] Failed to build global endpoints for port TCP/5000-tcp: empty IP address endpoints for service openshift-image-registry/image-registry I0429 06:48:58.937899 3947 pod.go:62] [openshift-console/downloads-5c54f4944f-ft449] pod update took 25.980795ms I0429 06:48:58.937929 3947 base_network_controller_pods.go:951] [default/openshift-console/downloads-5c54f4944f-ft449] addLogicalPort annotation time took 26.014666ms I0429 06:48:58.939127 3947 pods.go:271] [openshift-console/downloads-5c54f4944f-ft449] addLogicalPort took 27.29213ms, libovsdb time 869.423µs I0429 06:48:59.099712 3947 base_network_controller_pods.go:487] [default/openshift-insights/insights-runtime-extractor-qjchp] creating logical port openshift-insights_insights-runtime-extractor-qjchp for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:59.099880 3947 kube.go:256] Updating pod openshift-insights/insights-runtime-extractor-qjchp I0429 06:48:59.141849 3947 pod.go:62] [openshift-insights/insights-runtime-extractor-qjchp] pod update took 42.009142ms I0429 06:48:59.141883 3947 base_network_controller_pods.go:951] [default/openshift-insights/insights-runtime-extractor-qjchp] addLogicalPort annotation time took 42.048376ms I0429 06:48:59.143438 3947 pods.go:271] [openshift-insights/insights-runtime-extractor-qjchp] addLogicalPort took 43.752003ms, libovsdb time 1.104507ms I0429 06:48:59.270054 3947 cni.go:452] [openshift-console/downloads-5c54f4944f-ft449 73328f9c920976b732913654dad16acd13f79595bc486b9f94f0506ce5d48f46 network default NAD default NAD key ] ADD starting CNI request [openshift-console/downloads-5c54f4944f-ft449 73328f9c920976b732913654dad16acd13f79595bc486b9f94f0506ce5d48f46 network default NAD default NAD key ] I0429 06:48:59.283346 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: downloads-5c54f4944f-ft449, hostIfaceName: 73328f9c920976b, network: default, NAD default, SandboxID: "73328f9c920976b732913654dad16acd13f79595bc486b9f94f0506ce5d48f46", PCI device ID: , UID: "34e86f9f-3593-4a07-92ec-20440d3320ec", MAC: 0a:58:0a:81:00:09, IPs: [10.129.0.9/23] I0429 06:48:59.332113 3947 cni.go:473] [openshift-console/downloads-5c54f4944f-ft449 73328f9c920976b732913654dad16acd13f79595bc486b9f94f0506ce5d48f46 network default NAD default NAD key default] ADD finished CNI request [openshift-console/downloads-5c54f4944f-ft449 73328f9c920976b732913654dad16acd13f79595bc486b9f94f0506ce5d48f46 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"b2:7e:8c:45:cf:ad\",\"name\":\"73328f9c920976b\"},{\"mac\":\"0a:58:0a:81:00:09\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/79dce7b8-cf73-4123-915c-27dad61bac5f\"}],\"ips\":[{\"address\":\"10.129.0.9/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:48:59.457980 3947 cni.go:452] [openshift-insights/insights-runtime-extractor-qjchp 99bb104be7409d0eb32b644bdf0d2203638dce3e93ef4b02846351406371481c network default NAD default NAD key ] ADD starting CNI request [openshift-insights/insights-runtime-extractor-qjchp 99bb104be7409d0eb32b644bdf0d2203638dce3e93ef4b02846351406371481c network default NAD default NAD key ] I0429 06:48:59.470495 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-insights, podName: insights-runtime-extractor-qjchp, hostIfaceName: 99bb104be7409d0, network: default, NAD default, SandboxID: "99bb104be7409d0eb32b644bdf0d2203638dce3e93ef4b02846351406371481c", PCI device ID: , UID: "9cbfee6f-454d-4299-9253-62681433aa14", MAC: 0a:58:0a:81:00:0a, IPs: [10.129.0.10/23] I0429 06:48:59.518084 3947 cni.go:473] [openshift-insights/insights-runtime-extractor-qjchp 99bb104be7409d0eb32b644bdf0d2203638dce3e93ef4b02846351406371481c network default NAD default NAD key default] ADD finished CNI request [openshift-insights/insights-runtime-extractor-qjchp 99bb104be7409d0eb32b644bdf0d2203638dce3e93ef4b02846351406371481c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:6b:a9:2b:0a:2d\",\"name\":\"99bb104be7409d0\"},{\"mac\":\"0a:58:0a:81:00:0a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/089e858e-dbd6-45b5-9835-5b0f26ccacf2\"}],\"ips\":[{\"address\":\"10.129.0.10/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:48:59.626456 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0429 06:48:59.626511 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default W0429 06:48:59.742083 3947 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0429 06:48:59.742101 3947 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0429 06:48:59.742106 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default I0429 06:48:59.944909 3947 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-fxxbc] creating logical port openshift-monitoring_prometheus-operator-admission-webhook-59b79769fb-fxxbc for pod on switch ip-10-0-2-36.ec2.internal I0429 06:48:59.945007 3947 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-fxxbc I0429 06:48:59.972051 3947 pod.go:62] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-fxxbc] pod update took 27.07373ms I0429 06:48:59.972080 3947 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-fxxbc] addLogicalPort annotation time took 27.108609ms I0429 06:48:59.973336 3947 pods.go:271] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-fxxbc] addLogicalPort took 28.442569ms, libovsdb time 937.373µs W0429 06:49:00.619541 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:00.619561 3947 util.go:870] Failed to build global endpoints for port UDP/dns: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:00.619566 3947 util.go:870] Failed to build global endpoints for port TCP/dns-tcp: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:00.666117 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-default W0429 06:49:00.666146 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-default I0429 06:49:00.753559 3947 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT W0429 06:49:00.768164 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 80: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 06:49:00.768181 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 1936: empty IP address endpoints for service openshift-ingress/router-internal-default W0429 06:49:00.768186 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 443: empty IP address endpoints for service openshift-ingress/router-internal-default I0429 06:49:00.906399 3947 cni.go:452] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-fxxbc 150e0b64ec146cb400509325c048c2eb14ebb5fb79d26b89b43c20080c25eff3 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-fxxbc 150e0b64ec146cb400509325c048c2eb14ebb5fb79d26b89b43c20080c25eff3 network default NAD default NAD key ] I0429 06:49:00.916757 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-admission-webhook-59b79769fb-fxxbc, hostIfaceName: 150e0b64ec146cb, network: default, NAD default, SandboxID: "150e0b64ec146cb400509325c048c2eb14ebb5fb79d26b89b43c20080c25eff3", PCI device ID: , UID: "0f9dbd14-cc68-4d6e-b03a-b6f18904a5fd", MAC: 0a:58:0a:81:00:0b, IPs: [10.129.0.11/23] I0429 06:49:00.964773 3947 cni.go:473] [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-fxxbc 150e0b64ec146cb400509325c048c2eb14ebb5fb79d26b89b43c20080c25eff3 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-admission-webhook-59b79769fb-fxxbc 150e0b64ec146cb400509325c048c2eb14ebb5fb79d26b89b43c20080c25eff3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"12:1d:98:d1:f4:ea\",\"name\":\"150e0b64ec146cb\"},{\"mac\":\"0a:58:0a:81:00:0b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e6acac44-4303-4dd1-a018-86f07b2d06be\"}],\"ips\":[{\"address\":\"10.129.0.11/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:49:01.558409 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:01.558431 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:01.558439 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:01.558455 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:01.558462 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:01.558468 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:02.553990 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook W0429 06:49:02.687227 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-monitoring/prometheus-operator-admission-webhook I0429 06:49:03.109121 3947 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-operator-76b589d668-w9gg4] creating logical port openshift-monitoring_prometheus-operator-76b589d668-w9gg4 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:49:03.109225 3947 kube.go:256] Updating pod openshift-monitoring/prometheus-operator-76b589d668-w9gg4 I0429 06:49:03.123013 3947 pod.go:62] [openshift-monitoring/prometheus-operator-76b589d668-w9gg4] pod update took 13.815173ms I0429 06:49:03.123035 3947 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-operator-76b589d668-w9gg4] addLogicalPort annotation time took 13.84172ms I0429 06:49:03.124281 3947 pods.go:271] [openshift-monitoring/prometheus-operator-76b589d668-w9gg4] addLogicalPort took 15.18354ms, libovsdb time 882.269µs I0429 06:49:04.068368 3947 cni.go:452] [openshift-monitoring/prometheus-operator-76b589d668-w9gg4 f0d5ebb4e9af66f4ec3ede592986488a9a3250e85920261f5f9835c61b8384b0 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-operator-76b589d668-w9gg4 f0d5ebb4e9af66f4ec3ede592986488a9a3250e85920261f5f9835c61b8384b0 network default NAD default NAD key ] I0429 06:49:04.077977 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-operator-76b589d668-w9gg4, hostIfaceName: f0d5ebb4e9af66f, network: default, NAD default, SandboxID: "f0d5ebb4e9af66f4ec3ede592986488a9a3250e85920261f5f9835c61b8384b0", PCI device ID: , UID: "d97b7c15-9601-4cac-bcfb-190d34e9ed57", MAC: 0a:58:0a:81:00:0c, IPs: [10.129.0.12/23] I0429 06:49:04.127128 3947 cni.go:473] [openshift-monitoring/prometheus-operator-76b589d668-w9gg4 f0d5ebb4e9af66f4ec3ede592986488a9a3250e85920261f5f9835c61b8384b0 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-operator-76b589d668-w9gg4 f0d5ebb4e9af66f4ec3ede592986488a9a3250e85920261f5f9835c61b8384b0 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1e:48:e3:58:9e:c3\",\"name\":\"f0d5ebb4e9af66f\"},{\"mac\":\"0a:58:0a:81:00:0c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8b4f5de0-7083-4f64-9ba2-a58daeb1e9ab\"}],\"ips\":[{\"address\":\"10.129.0.12/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:49:04.322436 3947 base_network_controller_pods.go:487] [default/openshift-console/console-6cff5669b6-8s7k7] creating logical port openshift-console_console-6cff5669b6-8s7k7 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:49:04.322534 3947 kube.go:256] Updating pod openshift-console/console-6cff5669b6-8s7k7 I0429 06:49:04.341167 3947 pod.go:62] [openshift-console/console-6cff5669b6-8s7k7] pod update took 18.655478ms I0429 06:49:04.341189 3947 base_network_controller_pods.go:951] [default/openshift-console/console-6cff5669b6-8s7k7] addLogicalPort annotation time took 18.683473ms I0429 06:49:04.342353 3947 pods.go:271] [openshift-console/console-6cff5669b6-8s7k7] addLogicalPort took 19.93201ms, libovsdb time 839.08µs I0429 06:49:04.679371 3947 cni.go:452] [openshift-console/console-6cff5669b6-8s7k7 bcbba5f09f7254c8d9ab3126f6576cf3284badeab9029193ab38b94ea5c03e76 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6cff5669b6-8s7k7 bcbba5f09f7254c8d9ab3126f6576cf3284badeab9029193ab38b94ea5c03e76 network default NAD default NAD key ] I0429 06:49:04.690561 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6cff5669b6-8s7k7, hostIfaceName: bcbba5f09f7254c, network: default, NAD default, SandboxID: "bcbba5f09f7254c8d9ab3126f6576cf3284badeab9029193ab38b94ea5c03e76", PCI device ID: , UID: "ed4059af-4417-41e2-bf9b-fdc4d0628924", MAC: 0a:58:0a:81:00:0d, IPs: [10.129.0.13/23] I0429 06:49:04.738709 3947 cni.go:473] [openshift-console/console-6cff5669b6-8s7k7 bcbba5f09f7254c8d9ab3126f6576cf3284badeab9029193ab38b94ea5c03e76 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6cff5669b6-8s7k7 bcbba5f09f7254c8d9ab3126f6576cf3284badeab9029193ab38b94ea5c03e76 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:d4:af:3b:d3:8b\",\"name\":\"bcbba5f09f7254c\"},{\"mac\":\"0a:58:0a:81:00:0d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/3f9f5644-660c-490f-89f2-468edb128098\"}],\"ips\":[{\"address\":\"10.129.0.13/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:49:05.577182 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 06:49:05.577212 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 06:49:08.594013 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0429 06:49:08.665791 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0429 06:49:08.928977 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 06:49:08.929002 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 06:49:09.310160 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0429 06:49:09.704520 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:09.704543 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:09.704551 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:09.704568 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:09.704575 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:09.704601 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default I0429 06:49:10.039094 3947 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:49:10.039385 3947 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0429 06:49:10.060025 3947 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 20.731481ms I0429 06:49:10.060051 3947 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 20.761989ms I0429 06:49:10.061440 3947 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 22.370416ms, libovsdb time 1.025019ms I0429 06:49:10.413118 3947 cni.go:452] [openshift-monitoring/alertmanager-main-0 f44d927aebb42879374fe4fd9bf08f1c3dd2e0de57e8b2095606f4554d686139 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 f44d927aebb42879374fe4fd9bf08f1c3dd2e0de57e8b2095606f4554d686139 network default NAD default NAD key ] I0429 06:49:10.423149 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: f44d927aebb4287, network: default, NAD default, SandboxID: "f44d927aebb42879374fe4fd9bf08f1c3dd2e0de57e8b2095606f4554d686139", PCI device ID: , UID: "a26b0a1f-5758-431f-bd6c-e8f51ce99cba", MAC: 0a:58:0a:81:00:0e, IPs: [10.129.0.14/23] I0429 06:49:10.475014 3947 cni.go:473] [openshift-monitoring/alertmanager-main-0 f44d927aebb42879374fe4fd9bf08f1c3dd2e0de57e8b2095606f4554d686139 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 f44d927aebb42879374fe4fd9bf08f1c3dd2e0de57e8b2095606f4554d686139 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:0a:22:0a:4b:33\",\"name\":\"f44d927aebb4287\"},{\"mac\":\"0a:58:0a:81:00:0e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/dfc29be9-a320-4cf6-960f-69e9cd02e5b5\"}],\"ips\":[{\"address\":\"10.129.0.14/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:49:10.478880 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 06:49:10.478901 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 06:49:10.627076 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:10.627098 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5353: empty IP address endpoints for service openshift-dns/dns-default W0429 06:49:10.627105 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9154: empty IP address endpoints for service openshift-dns/dns-default I0429 06:49:11.922842 3947 base_network_controller_pods.go:487] [default/openshift-monitoring/thanos-querier-766b7f5496-8ghw5] creating logical port openshift-monitoring_thanos-querier-766b7f5496-8ghw5 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:49:11.922953 3947 kube.go:256] Updating pod openshift-monitoring/thanos-querier-766b7f5496-8ghw5 I0429 06:49:11.943932 3947 pod.go:62] [openshift-monitoring/thanos-querier-766b7f5496-8ghw5] pod update took 21.022353ms I0429 06:49:11.943953 3947 base_network_controller_pods.go:951] [default/openshift-monitoring/thanos-querier-766b7f5496-8ghw5] addLogicalPort annotation time took 21.047513ms I0429 06:49:11.945372 3947 pods.go:271] [openshift-monitoring/thanos-querier-766b7f5496-8ghw5] addLogicalPort took 22.55136ms, libovsdb time 1.012433ms W0429 06:49:12.772595 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console W0429 06:49:13.339677 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-console/console I0429 06:49:13.354772 3947 base_network_controller_pods.go:487] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-srvv5] creating logical port openshift-monitoring_monitoring-plugin-5cbb48bf6b-srvv5 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:49:13.354897 3947 kube.go:256] Updating pod openshift-monitoring/monitoring-plugin-5cbb48bf6b-srvv5 I0429 06:49:13.374328 3947 pod.go:62] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-srvv5] pod update took 19.453665ms I0429 06:49:13.374353 3947 base_network_controller_pods.go:951] [default/openshift-monitoring/monitoring-plugin-5cbb48bf6b-srvv5] addLogicalPort annotation time took 19.4806ms I0429 06:49:13.375503 3947 pods.go:271] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-srvv5] addLogicalPort took 20.74815ms, libovsdb time 802.696µs W0429 06:49:13.379565 3947 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-console/console-8545b5ddd-b2h69 IP address from the namespace address-set, err: pod openshift-console/console-8545b5ddd-b2h69: no pod IPs found I0429 06:49:13.566173 3947 base_network_controller_pods.go:487] [default/openshift-monitoring/metrics-server-7fff5cd4f4-xwptv] creating logical port openshift-monitoring_metrics-server-7fff5cd4f4-xwptv for pod on switch ip-10-0-2-36.ec2.internal I0429 06:49:13.566316 3947 kube.go:256] Updating pod openshift-monitoring/metrics-server-7fff5cd4f4-xwptv I0429 06:49:13.585175 3947 pod.go:62] [openshift-monitoring/metrics-server-7fff5cd4f4-xwptv] pod update took 18.881559ms I0429 06:49:13.585205 3947 base_network_controller_pods.go:951] [default/openshift-monitoring/metrics-server-7fff5cd4f4-xwptv] addLogicalPort annotation time took 18.914094ms I0429 06:49:13.586538 3947 pods.go:271] [openshift-monitoring/metrics-server-7fff5cd4f4-xwptv] addLogicalPort took 20.38358ms, libovsdb time 988.155µs I0429 06:49:15.180122 3947 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:49:15.180238 3947 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0429 06:49:15.207006 3947 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 26.816039ms I0429 06:49:15.207033 3947 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 26.850463ms I0429 06:49:15.208429 3947 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 28.32556ms, libovsdb time 1.038757ms W0429 06:49:17.661076 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 06:49:17.661101 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry I0429 06:49:18.295186 3947 cni.go:452] [openshift-monitoring/prometheus-k8s-0 9d43787c3fe3d5ca45df6af4af592dd1579c6018a272308c3900205028af630f network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 9d43787c3fe3d5ca45df6af4af592dd1579c6018a272308c3900205028af630f network default NAD default NAD key ] I0429 06:49:18.298000 3947 cni.go:452] [openshift-monitoring/thanos-querier-766b7f5496-8ghw5 a0f520ead6f597f2de08508c1378123c46d50c0c91e5de428b6aff4411fe0edc network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/thanos-querier-766b7f5496-8ghw5 a0f520ead6f597f2de08508c1378123c46d50c0c91e5de428b6aff4411fe0edc network default NAD default NAD key ] I0429 06:49:18.298000 3947 cni.go:452] [openshift-monitoring/metrics-server-7fff5cd4f4-xwptv baeeae3fb3b9d6a4edbd4cc4e399eed868260891e06bd9ef409fff8601bc3d87 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/metrics-server-7fff5cd4f4-xwptv baeeae3fb3b9d6a4edbd4cc4e399eed868260891e06bd9ef409fff8601bc3d87 network default NAD default NAD key ] I0429 06:49:18.298337 3947 cni.go:452] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-srvv5 9022e6965d100caf0dfa20ad254ab01e2525b6d19bd46376053d2623d62cffd1 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-srvv5 9022e6965d100caf0dfa20ad254ab01e2525b6d19bd46376053d2623d62cffd1 network default NAD default NAD key ] I0429 06:49:18.309780 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: 9d43787c3fe3d5c, network: default, NAD default, SandboxID: "9d43787c3fe3d5ca45df6af4af592dd1579c6018a272308c3900205028af630f", PCI device ID: , UID: "eab27cd3-b2dd-468a-be4d-ab8446f83c60", MAC: 0a:58:0a:81:00:12, IPs: [10.129.0.18/23] I0429 06:49:18.313110 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: thanos-querier-766b7f5496-8ghw5, hostIfaceName: a0f520ead6f597f, network: default, NAD default, SandboxID: "a0f520ead6f597f2de08508c1378123c46d50c0c91e5de428b6aff4411fe0edc", PCI device ID: , UID: "5e24d6b9-702c-4f19-a242-93aebed14bfa", MAC: 0a:58:0a:81:00:0f, IPs: [10.129.0.15/23] I0429 06:49:18.313741 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: monitoring-plugin-5cbb48bf6b-srvv5, hostIfaceName: 9022e6965d100ca, network: default, NAD default, SandboxID: "9022e6965d100caf0dfa20ad254ab01e2525b6d19bd46376053d2623d62cffd1", PCI device ID: , UID: "c19df215-1e4f-4ef0-9757-74064009344a", MAC: 0a:58:0a:81:00:10, IPs: [10.129.0.16/23] I0429 06:49:18.314114 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: metrics-server-7fff5cd4f4-xwptv, hostIfaceName: baeeae3fb3b9d6a, network: default, NAD default, SandboxID: "baeeae3fb3b9d6a4edbd4cc4e399eed868260891e06bd9ef409fff8601bc3d87", PCI device ID: , UID: "f1c76933-473f-42c2-95de-939174e78fea", MAC: 0a:58:0a:81:00:11, IPs: [10.129.0.17/23] I0429 06:49:18.374083 3947 cni.go:473] [openshift-monitoring/prometheus-k8s-0 9d43787c3fe3d5ca45df6af4af592dd1579c6018a272308c3900205028af630f network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 9d43787c3fe3d5ca45df6af4af592dd1579c6018a272308c3900205028af630f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"7a:17:f6:23:b8:c5\",\"name\":\"9d43787c3fe3d5c\"},{\"mac\":\"0a:58:0a:81:00:12\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d83b909d-78ba-47b3-913f-a8b0f250aa8d\"}],\"ips\":[{\"address\":\"10.129.0.18/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:49:18.389109 3947 cni.go:473] [openshift-monitoring/monitoring-plugin-5cbb48bf6b-srvv5 9022e6965d100caf0dfa20ad254ab01e2525b6d19bd46376053d2623d62cffd1 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/monitoring-plugin-5cbb48bf6b-srvv5 9022e6965d100caf0dfa20ad254ab01e2525b6d19bd46376053d2623d62cffd1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f6:0c:f7:f0:f7:49\",\"name\":\"9022e6965d100ca\"},{\"mac\":\"0a:58:0a:81:00:10\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/61b487e8-7af9-422f-809c-30671633e959\"}],\"ips\":[{\"address\":\"10.129.0.16/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:49:18.613652 3947 cni.go:473] [openshift-monitoring/metrics-server-7fff5cd4f4-xwptv baeeae3fb3b9d6a4edbd4cc4e399eed868260891e06bd9ef409fff8601bc3d87 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/metrics-server-7fff5cd4f4-xwptv baeeae3fb3b9d6a4edbd4cc4e399eed868260891e06bd9ef409fff8601bc3d87 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"a2:05:3e:0b:39:ec\",\"name\":\"baeeae3fb3b9d6a\"},{\"mac\":\"0a:58:0a:81:00:11\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/f9defffd-72b1-4d78-894a-852fc2b89ca9\"}],\"ips\":[{\"address\":\"10.129.0.17/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:49:18.613743 3947 cni.go:473] [openshift-monitoring/thanos-querier-766b7f5496-8ghw5 a0f520ead6f597f2de08508c1378123c46d50c0c91e5de428b6aff4411fe0edc network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/thanos-querier-766b7f5496-8ghw5 a0f520ead6f597f2de08508c1378123c46d50c0c91e5de428b6aff4411fe0edc network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ee:ad:cb:6b:ee:93\",\"name\":\"a0f520ead6f597f\"},{\"mac\":\"0a:58:0a:81:00:0f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6a4d91c6-8a6e-41e6-b2cf-a0fa9322e2fc\"}],\"ips\":[{\"address\":\"10.129.0.15/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:49:18.735742 3947 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:49:18.735769 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:49:18.906460 3947 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:49:18.906482 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:49:18.906487 3947 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:49:19.689529 3947 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:49:19.689552 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:49:19.771510 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0429 06:49:19.771628 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0429 06:49:19.782147 3947 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:49:19.782174 3947 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:49:19.782181 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:49:22.685503 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server W0429 06:49:23.819283 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-monitoring/metrics-server I0429 06:49:24.157478 3947 obj_retry.go:498] Detected object openshift-image-registry/image-registry-77769f5bc5-qtsp7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:49:24.158079 3947 obj_retry.go:498] Detected object openshift-image-registry/image-registry-77769f5bc5-qtsp7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0429 06:49:24.765157 3947 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:24.765177 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:24.765188 3947 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:24.765194 3947 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:24.796035 3947 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:24.796059 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:24.796066 3947 util.go:870] Failed to build global endpoints for port TCP/tenancy: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:24.796072 3947 util.go:870] Failed to build global endpoints for port TCP/tenancy-rules: empty IP address endpoints for service openshift-monitoring/thanos-querier I0429 06:49:24.808556 3947 obj_retry.go:492] Detected object openshift-image-registry/image-registry-77769f5bc5-qtsp7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:49:24.808579 3947 obj_retry.go:492] Detected object openshift-image-registry/image-registry-77769f5bc5-qtsp7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:49:24.817487 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-77769f5bc5-qtsp7 I0429 06:49:24.817517 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-77769f5bc5-qtsp7 I0429 06:49:25.757796 3947 cni.go:452] [openshift-image-registry/image-registry-76d6dbccdd-52h2m 6ce1d7cfff8106f1d69f306defdf81c1349ecd4b423c1bd12f1e3b8741d20f59 network default NAD default NAD key ] DEL starting CNI request [openshift-image-registry/image-registry-76d6dbccdd-52h2m 6ce1d7cfff8106f1d69f306defdf81c1349ecd4b423c1bd12f1e3b8741d20f59 network default NAD default NAD key ] I0429 06:49:25.842704 3947 cni.go:473] [openshift-image-registry/image-registry-76d6dbccdd-52h2m 6ce1d7cfff8106f1d69f306defdf81c1349ecd4b423c1bd12f1e3b8741d20f59 network default NAD default NAD key default] DEL finished CNI request [openshift-image-registry/image-registry-76d6dbccdd-52h2m 6ce1d7cfff8106f1d69f306defdf81c1349ecd4b423c1bd12f1e3b8741d20f59 network default NAD default NAD key default], result "{}", err I0429 06:49:25.897832 3947 obj_retry.go:498] Detected object openshift-image-registry/image-registry-76d6dbccdd-52h2m of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:49:25.897874 3947 pods.go:176] Deleting pod: openshift-image-registry/image-registry-76d6dbccdd-52h2m I0429 06:49:25.899361 3947 pods.go:236] Attempting to release IPs for pod: openshift-image-registry/image-registry-76d6dbccdd-52h2m, ips: 10.129.0.5 I0429 06:49:25.899392 3947 obj_retry.go:498] Detected object openshift-image-registry/image-registry-76d6dbccdd-52h2m of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:49:25.908147 3947 base_network_controller_pods.go:487] [default/openshift-image-registry/image-registry-777fb645d8-nfh2q] creating logical port openshift-image-registry_image-registry-777fb645d8-nfh2q for pod on switch ip-10-0-2-36.ec2.internal I0429 06:49:25.908237 3947 kube.go:256] Updating pod openshift-image-registry/image-registry-777fb645d8-nfh2q I0429 06:49:25.927349 3947 pod.go:62] [openshift-image-registry/image-registry-777fb645d8-nfh2q] pod update took 19.14166ms I0429 06:49:25.927381 3947 base_network_controller_pods.go:951] [default/openshift-image-registry/image-registry-777fb645d8-nfh2q] addLogicalPort annotation time took 19.177518ms I0429 06:49:25.928950 3947 pods.go:271] [openshift-image-registry/image-registry-777fb645d8-nfh2q] addLogicalPort took 20.820296ms, libovsdb time 1.119112ms I0429 06:49:26.289049 3947 cni.go:452] [openshift-image-registry/image-registry-777fb645d8-nfh2q 1e501e59a0ab346588b50b0afc9828f85a3ae73b1a7fcea7afcd61cdbb42ec9c network default NAD default NAD key ] ADD starting CNI request [openshift-image-registry/image-registry-777fb645d8-nfh2q 1e501e59a0ab346588b50b0afc9828f85a3ae73b1a7fcea7afcd61cdbb42ec9c network default NAD default NAD key ] I0429 06:49:26.300364 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-image-registry, podName: image-registry-777fb645d8-nfh2q, hostIfaceName: 1e501e59a0ab346, network: default, NAD default, SandboxID: "1e501e59a0ab346588b50b0afc9828f85a3ae73b1a7fcea7afcd61cdbb42ec9c", PCI device ID: , UID: "7a96c87b-b4b5-4f42-9551-ece967a9abe5", MAC: 0a:58:0a:81:00:13, IPs: [10.129.0.19/23] I0429 06:49:26.360853 3947 cni.go:473] [openshift-image-registry/image-registry-777fb645d8-nfh2q 1e501e59a0ab346588b50b0afc9828f85a3ae73b1a7fcea7afcd61cdbb42ec9c network default NAD default NAD key default] ADD finished CNI request [openshift-image-registry/image-registry-777fb645d8-nfh2q 1e501e59a0ab346588b50b0afc9828f85a3ae73b1a7fcea7afcd61cdbb42ec9c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4a:90:05:78:37:55\",\"name\":\"1e501e59a0ab346\"},{\"mac\":\"0a:58:0a:81:00:13\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6a1aa75f-17ae-470e-83e4-7bea2a14a757\"}],\"ips\":[{\"address\":\"10.129.0.19/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:49:26.779771 3947 obj_retry.go:492] Detected object openshift-image-registry/image-registry-76d6dbccdd-52h2m of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:49:26.779796 3947 obj_retry.go:492] Detected object openshift-image-registry/image-registry-76d6dbccdd-52h2m of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:49:26.788100 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-76d6dbccdd-52h2m I0429 06:49:26.788142 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-76d6dbccdd-52h2m W0429 06:49:30.619804 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service openshift-network-diagnostics/network-check-target W0429 06:49:30.736666 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.736686 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.736692 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.736700 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.736717 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.736723 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.736730 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.736736 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.785063 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9094: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.785085 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.785092 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9093: empty IP address endpoints for service openshift-monitoring/thanos-querier W0429 06:49:30.785108 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/thanos-querier I0429 06:49:35.902456 3947 obj_retry.go:498] Detected object openshift-console/console-6cff5669b6-hrhjw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:49:35.903206 3947 obj_retry.go:498] Detected object openshift-console/console-6cff5669b6-hrhjw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:49:36.760318 3947 obj_retry.go:492] Detected object openshift-console/console-6cff5669b6-hrhjw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:49:36.760343 3947 obj_retry.go:492] Detected object openshift-console/console-6cff5669b6-hrhjw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:49:36.762441 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6cff5669b6-hrhjw I0429 06:49:36.762465 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6cff5669b6-hrhjw I0429 06:49:38.507001 3947 cni.go:452] [openshift-console/console-6cff5669b6-8s7k7 bcbba5f09f7254c8d9ab3126f6576cf3284badeab9029193ab38b94ea5c03e76 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-6cff5669b6-8s7k7 bcbba5f09f7254c8d9ab3126f6576cf3284badeab9029193ab38b94ea5c03e76 network default NAD default NAD key ] I0429 06:49:38.593862 3947 cni.go:473] [openshift-console/console-6cff5669b6-8s7k7 bcbba5f09f7254c8d9ab3126f6576cf3284badeab9029193ab38b94ea5c03e76 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-6cff5669b6-8s7k7 bcbba5f09f7254c8d9ab3126f6576cf3284badeab9029193ab38b94ea5c03e76 network default NAD default NAD key default], result "{}", err I0429 06:49:38.655980 3947 obj_retry.go:498] Detected object openshift-console/console-6cff5669b6-8s7k7 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:49:38.656019 3947 pods.go:176] Deleting pod: openshift-console/console-6cff5669b6-8s7k7 I0429 06:49:38.657338 3947 pods.go:236] Attempting to release IPs for pod: openshift-console/console-6cff5669b6-8s7k7, ips: 10.129.0.13 I0429 06:49:38.657372 3947 obj_retry.go:498] Detected object openshift-console/console-6cff5669b6-8s7k7 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:49:38.667886 3947 base_network_controller_pods.go:487] [default/openshift-console/console-56cd459994-jngvh] creating logical port openshift-console_console-56cd459994-jngvh for pod on switch ip-10-0-2-36.ec2.internal I0429 06:49:38.667976 3947 kube.go:256] Updating pod openshift-console/console-56cd459994-jngvh I0429 06:49:38.682796 3947 pod.go:62] [openshift-console/console-56cd459994-jngvh] pod update took 14.843379ms I0429 06:49:38.682823 3947 base_network_controller_pods.go:951] [default/openshift-console/console-56cd459994-jngvh] addLogicalPort annotation time took 14.873666ms I0429 06:49:38.683938 3947 pods.go:271] [openshift-console/console-56cd459994-jngvh] addLogicalPort took 16.070444ms, libovsdb time 793.531µs I0429 06:49:38.822271 3947 obj_retry.go:492] Detected object openshift-console/console-6cff5669b6-8s7k7 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:49:38.822293 3947 obj_retry.go:492] Detected object openshift-console/console-6cff5669b6-8s7k7 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:49:38.826115 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-6cff5669b6-8s7k7 I0429 06:49:38.826142 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-6cff5669b6-8s7k7 I0429 06:49:39.038976 3947 cni.go:452] [openshift-console/console-56cd459994-jngvh 91f74cd53ae5c1fbe4637091cb57d5f8ae40f7f3957540e76d244f68dc065b3a network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-56cd459994-jngvh 91f74cd53ae5c1fbe4637091cb57d5f8ae40f7f3957540e76d244f68dc065b3a network default NAD default NAD key ] I0429 06:49:39.052760 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-56cd459994-jngvh, hostIfaceName: 91f74cd53ae5c1f, network: default, NAD default, SandboxID: "91f74cd53ae5c1fbe4637091cb57d5f8ae40f7f3957540e76d244f68dc065b3a", PCI device ID: , UID: "18ba96d0-6690-4b38-b936-400a6a55b8f8", MAC: 0a:58:0a:81:00:14, IPs: [10.129.0.20/23] I0429 06:49:39.106941 3947 cni.go:473] [openshift-console/console-56cd459994-jngvh 91f74cd53ae5c1fbe4637091cb57d5f8ae40f7f3957540e76d244f68dc065b3a network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-56cd459994-jngvh 91f74cd53ae5c1fbe4637091cb57d5f8ae40f7f3957540e76d244f68dc065b3a network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8a:bc:45:c4:2d:29\",\"name\":\"91f74cd53ae5c1f\"},{\"mac\":\"0a:58:0a:81:00:14\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6e5e82a8-8d8a-4ab9-8542-2c6d0a804430\"}],\"ips\":[{\"address\":\"10.129.0.20/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:49:43.702981 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0429 06:49:43.703010 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0429 06:49:44.769039 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 10250: empty IP address endpoints for service openshift-monitoring/metrics-server W0429 06:49:46.733806 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 06:49:46.733832 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 06:49:46.779842 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 06:49:46.779866 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 06:49:46.822338 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0429 06:49:46.822362 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0429 06:49:46.867761 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0429 06:49:46.867791 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0429 06:49:47.791157 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5000: empty IP address endpoints for service openshift-image-registry/image-registry W0429 06:49:49.882663 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console W0429 06:49:50.487833 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:49:50.487855 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:49:50.487861 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:49:54.883514 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-insights/metrics W0429 06:49:59.907372 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-service-ca-operator/metrics W0429 06:49:59.931666 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-kube-storage-version-migrator-operator/metrics I0429 06:50:12.177162 3947 obj_retry.go:498] Detected object openshift-console/console-8545b5ddd-vsqqk of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:12.177709 3947 obj_retry.go:498] Detected object openshift-console/console-8545b5ddd-vsqqk of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:12.965199 3947 obj_retry.go:492] Detected object openshift-console/console-8545b5ddd-vsqqk of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:12.965224 3947 obj_retry.go:492] Detected object openshift-console/console-8545b5ddd-vsqqk of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:12.976660 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-8545b5ddd-vsqqk I0429 06:50:12.976687 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-8545b5ddd-vsqqk I0429 06:50:13.064395 3947 obj_retry.go:498] Detected object openshift-image-registry/image-registry-76d6dbccdd-dsjbb of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:13.065089 3947 obj_retry.go:498] Detected object openshift-image-registry/image-registry-76d6dbccdd-dsjbb of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:13.338873 3947 obj_retry.go:498] Detected object openshift-ingress/router-default-57d97dff8f-95hdv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:13.339376 3947 obj_retry.go:498] Detected object openshift-ingress/router-default-57d97dff8f-95hdv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:13.971877 3947 obj_retry.go:492] Detected object openshift-image-registry/image-registry-76d6dbccdd-dsjbb of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:13.971909 3947 obj_retry.go:492] Detected object openshift-image-registry/image-registry-76d6dbccdd-dsjbb of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:13.981217 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-image-registry/image-registry-76d6dbccdd-dsjbb I0429 06:50:13.981248 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-image-registry/image-registry-76d6dbccdd-dsjbb I0429 06:50:14.010837 3947 obj_retry.go:492] Detected object openshift-ingress/router-default-57d97dff8f-95hdv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:14.010855 3947 obj_retry.go:492] Detected object openshift-ingress/router-default-57d97dff8f-95hdv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:14.013992 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-ingress/router-default-57d97dff8f-95hdv I0429 06:50:14.014013 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-ingress/router-default-57d97dff8f-95hdv W0429 06:50:15.970757 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:50:15.970782 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:50:15.970798 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:50:15.970806 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:50:16.027176 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:50:16.027195 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0429 06:50:30.710750 3947 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:30.711374 3947 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:30.994440 3947 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:30.994465 3947 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:31.003670 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-1 I0429 06:50:31.003690 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-1 I0429 06:50:35.640775 3947 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:35.641358 3947 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:36.036187 3947 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:36.036219 3947 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-1 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:36.045811 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-1 I0429 06:50:36.045839 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-1 I0429 06:50:40.200252 3947 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624090-8bk6z of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:40.200851 3947 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624090-8bk6z of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:50:41.084799 3947 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624090-8bk6z of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:50:41.084822 3947 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624090-8bk6z of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:02.629969 3947 cni.go:452] [openshift-monitoring/alertmanager-main-0 f44d927aebb42879374fe4fd9bf08f1c3dd2e0de57e8b2095606f4554d686139 network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/alertmanager-main-0 f44d927aebb42879374fe4fd9bf08f1c3dd2e0de57e8b2095606f4554d686139 network default NAD default NAD key ] I0429 06:51:02.706383 3947 cni.go:473] [openshift-monitoring/alertmanager-main-0 f44d927aebb42879374fe4fd9bf08f1c3dd2e0de57e8b2095606f4554d686139 network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/alertmanager-main-0 f44d927aebb42879374fe4fd9bf08f1c3dd2e0de57e8b2095606f4554d686139 network default NAD default NAD key default], result "{}", err I0429 06:51:02.769799 3947 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:02.769835 3947 pods.go:176] Deleting pod: openshift-monitoring/alertmanager-main-0 I0429 06:51:02.771113 3947 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/alertmanager-main-0, ips: 10.129.0.14 I0429 06:51:02.771143 3947 obj_retry.go:498] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:03.171983 3947 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:03.172018 3947 obj_retry.go:492] Detected object openshift-monitoring/alertmanager-main-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:03.182072 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/alertmanager-main-0 I0429 06:51:03.182111 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/alertmanager-main-0 I0429 06:51:03.230181 3947 base_network_controller_pods.go:487] [default/openshift-monitoring/alertmanager-main-0] creating logical port openshift-monitoring_alertmanager-main-0 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:51:03.230297 3947 kube.go:256] Updating pod openshift-monitoring/alertmanager-main-0 I0429 06:51:03.256671 3947 pod.go:62] [openshift-monitoring/alertmanager-main-0] pod update took 26.416263ms I0429 06:51:03.256696 3947 base_network_controller_pods.go:951] [default/openshift-monitoring/alertmanager-main-0] addLogicalPort annotation time took 26.445551ms I0429 06:51:03.257903 3947 pods.go:271] [openshift-monitoring/alertmanager-main-0] addLogicalPort took 27.740061ms, libovsdb time 886.208µs I0429 06:51:03.584539 3947 cni.go:452] [openshift-console/console-56cd459994-jngvh 91f74cd53ae5c1fbe4637091cb57d5f8ae40f7f3957540e76d244f68dc065b3a network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-56cd459994-jngvh 91f74cd53ae5c1fbe4637091cb57d5f8ae40f7f3957540e76d244f68dc065b3a network default NAD default NAD key ] I0429 06:51:03.588903 3947 cni.go:452] [openshift-monitoring/alertmanager-main-0 784b667f499ee80cc9d55be9393761aaaeba63f2a500e83bcd9c66b7ff045971 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/alertmanager-main-0 784b667f499ee80cc9d55be9393761aaaeba63f2a500e83bcd9c66b7ff045971 network default NAD default NAD key ] I0429 06:51:03.598437 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: alertmanager-main-0, hostIfaceName: 784b667f499ee80, network: default, NAD default, SandboxID: "784b667f499ee80cc9d55be9393761aaaeba63f2a500e83bcd9c66b7ff045971", PCI device ID: , UID: "79016296-5612-461e-beba-502f865bf76f", MAC: 0a:58:0a:81:00:15, IPs: [10.129.0.21/23] I0429 06:51:03.649621 3947 cni.go:473] [openshift-monitoring/alertmanager-main-0 784b667f499ee80cc9d55be9393761aaaeba63f2a500e83bcd9c66b7ff045971 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/alertmanager-main-0 784b667f499ee80cc9d55be9393761aaaeba63f2a500e83bcd9c66b7ff045971 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:25:d2:f8:66:38\",\"name\":\"784b667f499ee80\"},{\"mac\":\"0a:58:0a:81:00:15\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6220319a-0600-485c-94dc-95609d5d9869\"}],\"ips\":[{\"address\":\"10.129.0.21/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:51:03.657041 3947 cni.go:473] [openshift-console/console-56cd459994-jngvh 91f74cd53ae5c1fbe4637091cb57d5f8ae40f7f3957540e76d244f68dc065b3a network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-56cd459994-jngvh 91f74cd53ae5c1fbe4637091cb57d5f8ae40f7f3957540e76d244f68dc065b3a network default NAD default NAD key default], result "{}", err I0429 06:51:03.706343 3947 obj_retry.go:498] Detected object openshift-console/console-56cd459994-jngvh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:03.706373 3947 pods.go:176] Deleting pod: openshift-console/console-56cd459994-jngvh I0429 06:51:03.707840 3947 pods.go:236] Attempting to release IPs for pod: openshift-console/console-56cd459994-jngvh, ips: 10.129.0.20 I0429 06:51:03.707875 3947 obj_retry.go:498] Detected object openshift-console/console-56cd459994-jngvh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:03.718146 3947 base_network_controller_pods.go:487] [default/openshift-console/console-866d6b54c8-45cfw] creating logical port openshift-console_console-866d6b54c8-45cfw for pod on switch ip-10-0-2-36.ec2.internal I0429 06:51:03.718313 3947 kube.go:256] Updating pod openshift-console/console-866d6b54c8-45cfw I0429 06:51:03.733229 3947 pod.go:62] [openshift-console/console-866d6b54c8-45cfw] pod update took 14.982455ms I0429 06:51:03.733251 3947 base_network_controller_pods.go:951] [default/openshift-console/console-866d6b54c8-45cfw] addLogicalPort annotation time took 15.008336ms I0429 06:51:03.734705 3947 pods.go:271] [openshift-console/console-866d6b54c8-45cfw] addLogicalPort took 16.571007ms, libovsdb time 1.063835ms I0429 06:51:04.080760 3947 cni.go:452] [openshift-console/console-866d6b54c8-45cfw 001db0cae5775a159a1dda51c4ecefd120e455c716aff3386e969c52f6933f09 network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-866d6b54c8-45cfw 001db0cae5775a159a1dda51c4ecefd120e455c716aff3386e969c52f6933f09 network default NAD default NAD key ] I0429 06:51:04.090395 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-866d6b54c8-45cfw, hostIfaceName: 001db0cae5775a1, network: default, NAD default, SandboxID: "001db0cae5775a159a1dda51c4ecefd120e455c716aff3386e969c52f6933f09", PCI device ID: , UID: "0fd863e7-9b76-41db-889f-d12c67d836d1", MAC: 0a:58:0a:81:00:16, IPs: [10.129.0.22/23] I0429 06:51:04.138111 3947 cni.go:473] [openshift-console/console-866d6b54c8-45cfw 001db0cae5775a159a1dda51c4ecefd120e455c716aff3386e969c52f6933f09 network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-866d6b54c8-45cfw 001db0cae5775a159a1dda51c4ecefd120e455c716aff3386e969c52f6933f09 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0e:24:eb:45:ae:de\",\"name\":\"001db0cae5775a1\"},{\"mac\":\"0a:58:0a:81:00:16\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/791ff626-9145-4c4b-ad92-e72220e9fe10\"}],\"ips\":[{\"address\":\"10.129.0.22/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:51:04.212054 3947 obj_retry.go:492] Detected object openshift-console/console-56cd459994-jngvh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:04.212078 3947 obj_retry.go:492] Detected object openshift-console/console-56cd459994-jngvh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:04.223299 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-56cd459994-jngvh I0429 06:51:04.223332 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-56cd459994-jngvh W0429 06:51:14.265852 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0429 06:51:14.437303 3947 obj_retry.go:498] Detected object openshift-console/console-56cd459994-fthtx of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:14.438012 3947 obj_retry.go:498] Detected object openshift-console/console-56cd459994-fthtx of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:15.227897 3947 obj_retry.go:492] Detected object openshift-console/console-56cd459994-fthtx of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:15.227921 3947 obj_retry.go:492] Detected object openshift-console/console-56cd459994-fthtx of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:15.239721 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-56cd459994-fthtx I0429 06:51:15.239743 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-56cd459994-fthtx I0429 06:51:18.355670 3947 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db4rrt of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:18.356218 3947 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db4rrt of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:18.356249 3947 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db4rrt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:51:18.356259 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... I0429 06:51:18.356281 3947 obj_retry.go:498] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db4rrt of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:51:18.356292 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/deployment-validation-operator-catalog-unpack-bundles to delete 1 local pods... I0429 06:51:19.384928 3947 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db4rrt of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:19.384949 3947 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db4rrt of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:19.384963 3947 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db4rrt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:19.384974 3947 obj_retry.go:492] Detected object openshift-deployment-validation-operator/99b07c45bd4393a793940f31bf2887fd12a636d02cd8e5e017ce352a0db4rrt of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:21.301991 3947 namespace.go:142] [openshift-deployment-validation-operator] updating namespace I0429 06:51:21.312821 3947 namespace.go:142] [openshift-deployment-validation-operator] updating namespace W0429 06:51:33.656358 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9095: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:51:33.656384 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/alertmanager-main W0429 06:51:33.656393 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9097: empty IP address endpoints for service openshift-monitoring/alertmanager-main I0429 06:51:38.526887 3947 cni.go:452] [openshift-monitoring/prometheus-k8s-0 9d43787c3fe3d5ca45df6af4af592dd1579c6018a272308c3900205028af630f network default NAD default NAD key ] DEL starting CNI request [openshift-monitoring/prometheus-k8s-0 9d43787c3fe3d5ca45df6af4af592dd1579c6018a272308c3900205028af630f network default NAD default NAD key ] I0429 06:51:38.607435 3947 cni.go:473] [openshift-monitoring/prometheus-k8s-0 9d43787c3fe3d5ca45df6af4af592dd1579c6018a272308c3900205028af630f network default NAD default NAD key default] DEL finished CNI request [openshift-monitoring/prometheus-k8s-0 9d43787c3fe3d5ca45df6af4af592dd1579c6018a272308c3900205028af630f network default NAD default NAD key default], result "{}", err I0429 06:51:38.672266 3947 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:38.672308 3947 pods.go:176] Deleting pod: openshift-monitoring/prometheus-k8s-0 I0429 06:51:38.673689 3947 pods.go:236] Attempting to release IPs for pod: openshift-monitoring/prometheus-k8s-0, ips: 10.129.0.18 I0429 06:51:38.673722 3947 obj_retry.go:498] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:51:39.341553 3947 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:39.341604 3947 obj_retry.go:492] Detected object openshift-monitoring/prometheus-k8s-0 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:51:39.350413 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-monitoring/prometheus-k8s-0 I0429 06:51:39.350443 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-monitoring/prometheus-k8s-0 I0429 06:51:39.392464 3947 base_network_controller_pods.go:487] [default/openshift-monitoring/prometheus-k8s-0] creating logical port openshift-monitoring_prometheus-k8s-0 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:51:39.392621 3947 kube.go:256] Updating pod openshift-monitoring/prometheus-k8s-0 I0429 06:51:39.413510 3947 pod.go:62] [openshift-monitoring/prometheus-k8s-0] pod update took 20.939641ms I0429 06:51:39.413534 3947 base_network_controller_pods.go:951] [default/openshift-monitoring/prometheus-k8s-0] addLogicalPort annotation time took 20.968292ms I0429 06:51:39.414885 3947 pods.go:271] [openshift-monitoring/prometheus-k8s-0] addLogicalPort took 22.438041ms, libovsdb time 959.409µs I0429 06:51:39.752679 3947 cni.go:452] [openshift-monitoring/prometheus-k8s-0 a1c9a76328b6d9f753cd58e57354ac606ac7947fb344648118f5554aaa4d5255 network default NAD default NAD key ] ADD starting CNI request [openshift-monitoring/prometheus-k8s-0 a1c9a76328b6d9f753cd58e57354ac606ac7947fb344648118f5554aaa4d5255 network default NAD default NAD key ] I0429 06:51:39.762368 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-monitoring, podName: prometheus-k8s-0, hostIfaceName: a1c9a76328b6d9f, network: default, NAD default, SandboxID: "a1c9a76328b6d9f753cd58e57354ac606ac7947fb344648118f5554aaa4d5255", PCI device ID: , UID: "fb791a49-d23b-46d0-afcf-04c9ed5699b9", MAC: 0a:58:0a:81:00:17, IPs: [10.129.0.23/23] I0429 06:51:39.811646 3947 cni.go:473] [openshift-monitoring/prometheus-k8s-0 a1c9a76328b6d9f753cd58e57354ac606ac7947fb344648118f5554aaa4d5255 network default NAD default NAD key default] ADD finished CNI request [openshift-monitoring/prometheus-k8s-0 a1c9a76328b6d9f753cd58e57354ac606ac7947fb344648118f5554aaa4d5255 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"aa:a1:3c:b4:01:66\",\"name\":\"a1c9a76328b6d9f\"},{\"mac\":\"0a:58:0a:81:00:17\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6bbd36e4-b533-4f86-8978-77921f30cb5b\"}],\"ips\":[{\"address\":\"10.129.0.23/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:52:09.111862 3947 namespace.go:142] [openshift-kube-storage-version-migrator] updating namespace W0429 06:52:40.618687 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9092: empty IP address endpoints for service openshift-monitoring/prometheus-k8s W0429 06:52:40.618712 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9091: empty IP address endpoints for service openshift-monitoring/prometheus-k8s I0429 06:53:22.623157 3947 cni.go:452] [openshift-console/console-866d6b54c8-45cfw 001db0cae5775a159a1dda51c4ecefd120e455c716aff3386e969c52f6933f09 network default NAD default NAD key ] DEL starting CNI request [openshift-console/console-866d6b54c8-45cfw 001db0cae5775a159a1dda51c4ecefd120e455c716aff3386e969c52f6933f09 network default NAD default NAD key ] I0429 06:53:22.696294 3947 cni.go:473] [openshift-console/console-866d6b54c8-45cfw 001db0cae5775a159a1dda51c4ecefd120e455c716aff3386e969c52f6933f09 network default NAD default NAD key default] DEL finished CNI request [openshift-console/console-866d6b54c8-45cfw 001db0cae5775a159a1dda51c4ecefd120e455c716aff3386e969c52f6933f09 network default NAD default NAD key default], result "{}", err I0429 06:53:22.747467 3947 obj_retry.go:498] Detected object openshift-console/console-866d6b54c8-45cfw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:53:22.747502 3947 pods.go:176] Deleting pod: openshift-console/console-866d6b54c8-45cfw I0429 06:53:22.748671 3947 pods.go:236] Attempting to release IPs for pod: openshift-console/console-866d6b54c8-45cfw, ips: 10.129.0.22 I0429 06:53:22.748699 3947 obj_retry.go:498] Detected object openshift-console/console-866d6b54c8-45cfw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:53:22.756227 3947 base_network_controller_pods.go:487] [default/openshift-console/console-6b7b945ddd-7x58t] creating logical port openshift-console_console-6b7b945ddd-7x58t for pod on switch ip-10-0-2-36.ec2.internal I0429 06:53:22.756335 3947 kube.go:256] Updating pod openshift-console/console-6b7b945ddd-7x58t I0429 06:53:22.774303 3947 pod.go:62] [openshift-console/console-6b7b945ddd-7x58t] pod update took 17.998087ms I0429 06:53:22.774327 3947 base_network_controller_pods.go:951] [default/openshift-console/console-6b7b945ddd-7x58t] addLogicalPort annotation time took 18.024563ms I0429 06:53:22.775707 3947 pods.go:271] [openshift-console/console-6b7b945ddd-7x58t] addLogicalPort took 19.517827ms, libovsdb time 1.055863ms I0429 06:53:23.115278 3947 cni.go:452] [openshift-console/console-6b7b945ddd-7x58t f6da430400afbb722a5003045234ebe37a74b9cc43a0882295dc5721cb638f6f network default NAD default NAD key ] ADD starting CNI request [openshift-console/console-6b7b945ddd-7x58t f6da430400afbb722a5003045234ebe37a74b9cc43a0882295dc5721cb638f6f network default NAD default NAD key ] I0429 06:53:23.125405 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-console, podName: console-6b7b945ddd-7x58t, hostIfaceName: f6da430400afbb7, network: default, NAD default, SandboxID: "f6da430400afbb722a5003045234ebe37a74b9cc43a0882295dc5721cb638f6f", PCI device ID: , UID: "beadba56-8695-46f2-b184-81913510ab4a", MAC: 0a:58:0a:81:00:18, IPs: [10.129.0.24/23] I0429 06:53:23.174982 3947 cni.go:473] [openshift-console/console-6b7b945ddd-7x58t f6da430400afbb722a5003045234ebe37a74b9cc43a0882295dc5721cb638f6f network default NAD default NAD key default] ADD finished CNI request [openshift-console/console-6b7b945ddd-7x58t f6da430400afbb722a5003045234ebe37a74b9cc43a0882295dc5721cb638f6f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"de:3f:4d:f8:5c:57\",\"name\":\"f6da430400afbb7\"},{\"mac\":\"0a:58:0a:81:00:18\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/53f7bcd5-e971-4957-a537-ef00dc4c32df\"}],\"ips\":[{\"address\":\"10.129.0.24/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:53:23.757987 3947 obj_retry.go:492] Detected object openshift-console/console-866d6b54c8-45cfw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:53:23.758019 3947 obj_retry.go:492] Detected object openshift-console/console-866d6b54c8-45cfw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:53:23.768178 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-866d6b54c8-45cfw I0429 06:53:23.768212 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-866d6b54c8-45cfw W0429 06:53:33.792086 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-console/console I0429 06:53:34.039366 3947 obj_retry.go:498] Detected object openshift-console/console-866d6b54c8-4pmkn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:53:34.039907 3947 obj_retry.go:498] Detected object openshift-console/console-866d6b54c8-4pmkn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:53:34.704395 3947 obj_retry.go:492] Detected object openshift-console/console-866d6b54c8-4pmkn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:53:34.704431 3947 obj_retry.go:492] Detected object openshift-console/console-866d6b54c8-4pmkn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:53:34.714630 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod openshift-console/console-866d6b54c8-4pmkn I0429 06:53:34.714657 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod openshift-console/console-866d6b54c8-4pmkn I0429 06:53:47.064723 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=14 I0429 06:53:53.062155 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Node" totalItems=60 I0429 06:53:59.176893 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=6 I0429 06:54:39.059955 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=144 I0429 06:55:16.060220 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0429 06:55:59.148556 3947 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.BaselineAdminNetworkPolicy" totalItems=7 I0429 06:56:10.063236 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=72 I0429 06:56:18.148619 3947 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=8 I0429 06:56:21.716477 3947 namespace.go:98] [tsf] adding namespace I0429 06:56:21.717337 3947 namespace.go:102] [tsf] adding namespace took 833.106µs I0429 06:56:21.733343 3947 namespace.go:142] [tsf] updating namespace I0429 06:56:21.746882 3947 namespace.go:142] [tsf] updating namespace I0429 06:56:25.976807 3947 namespace.go:98] [tsf-tas] adding namespace I0429 06:56:25.977735 3947 namespace.go:102] [tsf-tas] adding namespace took 901.635µs I0429 06:56:25.978048 3947 namespace.go:98] [rhtpa-operator] adding namespace I0429 06:56:25.978667 3947 namespace.go:102] [rhtpa-operator] adding namespace took 604.572µs I0429 06:56:25.981661 3947 namespace.go:98] [tsf-keycloak] adding namespace I0429 06:56:25.982282 3947 namespace.go:102] [tsf-keycloak] adding namespace took 609.002µs I0429 06:56:25.982602 3947 namespace.go:98] [konflux-ui] adding namespace I0429 06:56:25.983245 3947 namespace.go:102] [konflux-ui] adding namespace took 628.365µs I0429 06:56:25.983333 3947 namespace.go:98] [tsf-tpa] adding namespace I0429 06:56:25.983336 3947 namespace.go:98] [cert-manager-operator] adding namespace I0429 06:56:25.983478 3947 namespace.go:98] [rhbk-operator] adding namespace I0429 06:56:25.983888 3947 namespace.go:102] [tsf-tpa] adding namespace took 543.872µs I0429 06:56:25.984508 3947 namespace.go:102] [cert-manager-operator] adding namespace took 1.157034ms I0429 06:56:25.985042 3947 namespace.go:102] [rhbk-operator] adding namespace took 1.551414ms I0429 06:56:25.985064 3947 namespace.go:98] [konflux-operator] adding namespace I0429 06:56:25.985520 3947 namespace.go:102] [konflux-operator] adding namespace took 450.105µs I0429 06:56:26.002409 3947 namespace.go:142] [tsf-tas] updating namespace I0429 06:56:26.068039 3947 namespace.go:142] [tsf-tas] updating namespace I0429 06:56:26.108711 3947 namespace.go:142] [tsf-keycloak] updating namespace I0429 06:56:26.125975 3947 namespace.go:142] [konflux-ui] updating namespace I0429 06:56:26.131728 3947 namespace.go:142] [cert-manager-operator] updating namespace I0429 06:56:26.139945 3947 namespace.go:142] [rhtpa-operator] updating namespace I0429 06:56:26.191725 3947 namespace.go:142] [tsf-tpa] updating namespace I0429 06:56:26.197909 3947 namespace.go:142] [rhbk-operator] updating namespace I0429 06:56:26.213095 3947 namespace.go:142] [konflux-operator] updating namespace I0429 06:56:26.226758 3947 namespace.go:142] [tsf-tas] updating namespace I0429 06:56:26.307491 3947 namespace.go:142] [tsf-keycloak] updating namespace W0429 06:56:26.319264 3947 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-deployment-validation-operator/deployment-validation-operator-metrics I0429 06:56:26.341707 3947 namespace.go:142] [tsf-keycloak] updating namespace I0429 06:56:26.451920 3947 namespace.go:142] [konflux-ui] updating namespace I0429 06:56:26.465060 3947 namespace.go:142] [konflux-ui] updating namespace I0429 06:56:26.484049 3947 namespace.go:142] [cert-manager-operator] updating namespace I0429 06:56:26.499332 3947 namespace.go:142] [cert-manager-operator] updating namespace I0429 06:56:26.516682 3947 namespace.go:142] [tsf-tpa] updating namespace I0429 06:56:26.533873 3947 namespace.go:142] [tsf-tpa] updating namespace I0429 06:56:26.552917 3947 namespace.go:142] [rhbk-operator] updating namespace I0429 06:56:26.566006 3947 namespace.go:142] [rhbk-operator] updating namespace I0429 06:56:26.575854 3947 namespace.go:142] [konflux-operator] updating namespace I0429 06:56:26.585539 3947 namespace.go:142] [konflux-operator] updating namespace I0429 06:56:26.595963 3947 namespace.go:142] [rhtpa-operator] updating namespace I0429 06:56:26.605617 3947 namespace.go:142] [rhtpa-operator] updating namespace I0429 06:56:27.276845 3947 namespace.go:142] [tsf-keycloak] updating namespace I0429 06:56:27.286892 3947 namespace.go:142] [tsf-tpa] updating namespace I0429 06:56:30.073338 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Pod" totalItems=635 I0429 06:56:34.719688 3947 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2pksmf of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:34.720298 3947 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2pksmf of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:34.720321 3947 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2pksmf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:34.720328 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 06:56:34.720338 3947 obj_retry.go:498] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2pksmf of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:34.720342 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 06:56:34.752709 3947 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51mvxt9 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:34.753218 3947 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51mvxt9 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:34.753238 3947 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51mvxt9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:34.753245 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 06:56:34.753254 3947 obj_retry.go:498] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51mvxt9 of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:34.753257 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 06:56:35.751268 3947 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2pksmf of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:35.751291 3947 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2pksmf of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:35.751302 3947 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2pksmf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:35.751313 3947 obj_retry.go:492] Detected object openshift-marketplace/a54ee9eda2828a12414ea519ebe1984caa1e89a473cf1b21256364ccf2pksmf of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:35.792506 3947 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51mvxt9 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:35.792525 3947 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51mvxt9 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:35.792537 3947 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51mvxt9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:35.792546 3947 obj_retry.go:492] Detected object openshift-marketplace/79f871b6353a8442b4f467c124caf7fa84891a4baa69c2582c3b4a7b51mvxt9 of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:36.164958 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=10 I0429 06:56:37.367557 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-deployment-validation-operator/allow-from-openshift-insights to delete 1 local pods... W0429 06:56:37.367636 3947 base_network_controller_namespace.go:458] Unable to remove remote zone pod's openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-2n4q6 IP address from the namespace address-set, err: pod openshift-deployment-validation-operator/deployment-validation-operator-65d68cbd6c-2n4q6: no pod IPs found I0429 06:56:41.587169 3947 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lv9sw of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:41.587718 3947 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lv9sw of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:41.587742 3947 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lv9sw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:41.587750 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 06:56:41.587760 3947 obj_retry.go:498] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lv9sw of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:41.587765 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 06:56:42.557470 3947 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7pltcv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:42.558028 3947 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7pltcv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:42.558059 3947 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7pltcv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:42.558068 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 06:56:42.558082 3947 obj_retry.go:498] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7pltcv of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:42.558089 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 06:56:42.629301 3947 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lv9sw of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:42.629324 3947 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lv9sw of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:42.629338 3947 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lv9sw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:42.629350 3947 obj_retry.go:492] Detected object openshift-marketplace/26f31e01798d5ff60e3a8dd38c21154e52540a0e6813a6498bf5aa0158lv9sw of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:43.584770 3947 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7pltcv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:43.584795 3947 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7pltcv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:43.584816 3947 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7pltcv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:43.584830 3947 obj_retry.go:492] Detected object openshift-marketplace/09e84d3fdfac1305b8f399db3dfe1b7c79d582fe1356304a82b56f71d7pltcv of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:44.630926 3947 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6b66t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:44.631506 3947 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6b66t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:44.631539 3947 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6b66t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:44.631550 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 06:56:44.631561 3947 obj_retry.go:498] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6b66t of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:44.631566 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 06:56:44.681792 3947 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136hsdlz of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:44.682266 3947 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136hsdlz of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:56:44.682295 3947 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136hsdlz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:44.682303 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/default-deny-all to delete 1 local pods... I0429 06:56:44.682317 3947 obj_retry.go:498] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136hsdlz of type *factory.localPodSelector in terminal state (e.g. completed) during update event: will remove it I0429 06:56:44.682326 3947 base_network_controller_policy.go:642] Processing NetworkPolicy openshift-marketplace/unpack-bundles to delete 1 local pods... I0429 06:56:45.650964 3947 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6b66t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:45.650986 3947 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6b66t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:45.651003 3947 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6b66t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:45.651016 3947 obj_retry.go:492] Detected object openshift-marketplace/2bb52b92bd31ddf2ebbc335370ac517be50e61a93c2fe375393413c19d6b66t of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:45.703854 3947 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136hsdlz of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:45.703875 3947 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136hsdlz of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:45.703886 3947 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136hsdlz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:56:45.703900 3947 obj_retry.go:492] Detected object openshift-marketplace/03dabd08ddba15f575168653b704f77a8a8f37b7d692dad9ab1e8c2136hsdlz of type *factory.localPodSelector in terminal state (e.g. completed) will be ignored as it has already been processed W0429 06:56:48.583098 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0429 06:56:51.503138 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0429 06:56:51.513395 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0429 06:56:51.520322 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service W0429 06:56:51.528438 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-tpa/rhtpa-operator-controller-manager-metrics-service I0429 06:57:00.895817 3947 base_network_controller_pods.go:487] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-5mkxr] creating logical port cert-manager-operator_cert-manager-operator-controller-manager-5b66478c58-5mkxr for pod on switch ip-10-0-2-36.ec2.internal I0429 06:57:00.895964 3947 kube.go:256] Updating pod cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-5mkxr I0429 06:57:00.933459 3947 pod.go:62] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-5mkxr] pod update took 37.521321ms I0429 06:57:00.933483 3947 base_network_controller_pods.go:951] [default/cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-5mkxr] addLogicalPort annotation time took 37.550542ms I0429 06:57:00.934636 3947 pods.go:271] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-5mkxr] addLogicalPort took 38.836657ms, libovsdb time 828.116µs I0429 06:57:01.260227 3947 cni.go:452] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-5mkxr 62003dffec108acf7d108294b580f131523940bf77f2c319d65a981b358815be network default NAD default NAD key ] ADD starting CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-5mkxr 62003dffec108acf7d108294b580f131523940bf77f2c319d65a981b358815be network default NAD default NAD key ] I0429 06:57:01.269551 3947 helper_linux.go:556] ConfigureOVS: namespace: cert-manager-operator, podName: cert-manager-operator-controller-manager-5b66478c58-5mkxr, hostIfaceName: 62003dffec108ac, network: default, NAD default, SandboxID: "62003dffec108acf7d108294b580f131523940bf77f2c319d65a981b358815be", PCI device ID: , UID: "db48cbf3-504b-4502-b1fd-7265b8f2c1b3", MAC: 0a:58:0a:81:00:19, IPs: [10.129.0.25/23] I0429 06:57:01.317936 3947 cni.go:473] [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-5mkxr 62003dffec108acf7d108294b580f131523940bf77f2c319d65a981b358815be network default NAD default NAD key default] ADD finished CNI request [cert-manager-operator/cert-manager-operator-controller-manager-5b66478c58-5mkxr 62003dffec108acf7d108294b580f131523940bf77f2c319d65a981b358815be network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ae:b0:8a:23:bd:38\",\"name\":\"62003dffec108ac\"},{\"mac\":\"0a:58:0a:81:00:19\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/027dcd09-62f5-433c-9d9b-db69fc1b49f2\"}],\"ips\":[{\"address\":\"10.129.0.25/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:57:01.957343 3947 namespace.go:142] [openshift-operators] updating namespace I0429 06:57:01.975112 3947 namespace.go:142] [openshift-operators] updating namespace I0429 06:57:05.187726 3947 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=9 I0429 06:57:05.364822 3947 namespace.go:98] [cert-manager] adding namespace I0429 06:57:05.365709 3947 namespace.go:102] [cert-manager] adding namespace took 867.452µs I0429 06:57:05.383749 3947 base_network_controller_pods.go:487] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-9wrf4] creating logical port konflux-operator_konflux-operator-controller-manager-66b879f574-9wrf4 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:57:05.383873 3947 kube.go:256] Updating pod konflux-operator/konflux-operator-controller-manager-66b879f574-9wrf4 I0429 06:57:05.400917 3947 namespace.go:142] [cert-manager] updating namespace I0429 06:57:05.424965 3947 pod.go:62] [konflux-operator/konflux-operator-controller-manager-66b879f574-9wrf4] pod update took 41.11571ms I0429 06:57:05.424994 3947 base_network_controller_pods.go:951] [default/konflux-operator/konflux-operator-controller-manager-66b879f574-9wrf4] addLogicalPort annotation time took 41.149423ms I0429 06:57:05.426304 3947 pods.go:271] [konflux-operator/konflux-operator-controller-manager-66b879f574-9wrf4] addLogicalPort took 42.571105ms, libovsdb time 884.69µs I0429 06:57:05.431769 3947 namespace.go:142] [cert-manager] updating namespace I0429 06:57:05.743562 3947 cni.go:452] [konflux-operator/konflux-operator-controller-manager-66b879f574-9wrf4 b27496517b253a3f4cd3bd7b723a8ac46cea10b88d06c70e63ceab170129a65c network default NAD default NAD key ] ADD starting CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-9wrf4 b27496517b253a3f4cd3bd7b723a8ac46cea10b88d06c70e63ceab170129a65c network default NAD default NAD key ] I0429 06:57:05.753198 3947 helper_linux.go:556] ConfigureOVS: namespace: konflux-operator, podName: konflux-operator-controller-manager-66b879f574-9wrf4, hostIfaceName: b27496517b253a3, network: default, NAD default, SandboxID: "b27496517b253a3f4cd3bd7b723a8ac46cea10b88d06c70e63ceab170129a65c", PCI device ID: , UID: "0ebb6217-8d4c-4025-8fb0-c56910148478", MAC: 0a:58:0a:81:00:1a, IPs: [10.129.0.26/23] I0429 06:57:05.803175 3947 cni.go:473] [konflux-operator/konflux-operator-controller-manager-66b879f574-9wrf4 b27496517b253a3f4cd3bd7b723a8ac46cea10b88d06c70e63ceab170129a65c network default NAD default NAD key default] ADD finished CNI request [konflux-operator/konflux-operator-controller-manager-66b879f574-9wrf4 b27496517b253a3f4cd3bd7b723a8ac46cea10b88d06c70e63ceab170129a65c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"92:0d:6e:b0:c4:69\",\"name\":\"b27496517b253a3\"},{\"mac\":\"0a:58:0a:81:00:1a\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/b9373dc3-6dc5-48a2-b63d-5e5accbacb5b\"}],\"ips\":[{\"address\":\"10.129.0.26/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:57:06.153099 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=10 W0429 06:57:08.619221 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0429 06:57:10.124920 3947 namespace.go:142] [openshift-operators] updating namespace I0429 06:57:10.824207 3947 base_network_controller_pods.go:487] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-b8cq9] creating logical port openshift-operators_openshift-pipelines-operator-6bdf6c8794-b8cq9 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:57:10.824324 3947 kube.go:256] Updating pod openshift-operators/openshift-pipelines-operator-6bdf6c8794-b8cq9 I0429 06:57:10.854468 3947 pod.go:62] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-b8cq9] pod update took 30.173327ms I0429 06:57:10.854487 3947 base_network_controller_pods.go:951] [default/openshift-operators/openshift-pipelines-operator-6bdf6c8794-b8cq9] addLogicalPort annotation time took 30.19623ms I0429 06:57:10.855636 3947 pods.go:271] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-b8cq9] addLogicalPort took 31.445969ms, libovsdb time 826.163µs I0429 06:57:10.931898 3947 base_network_controller_pods.go:487] [default/openshift-operators/tekton-operator-webhook-5864bd5898-79wzz] creating logical port openshift-operators_tekton-operator-webhook-5864bd5898-79wzz for pod on switch ip-10-0-2-36.ec2.internal I0429 06:57:10.932005 3947 kube.go:256] Updating pod openshift-operators/tekton-operator-webhook-5864bd5898-79wzz I0429 06:57:10.958177 3947 pod.go:62] [openshift-operators/tekton-operator-webhook-5864bd5898-79wzz] pod update took 26.193043ms I0429 06:57:10.958200 3947 base_network_controller_pods.go:951] [default/openshift-operators/tekton-operator-webhook-5864bd5898-79wzz] addLogicalPort annotation time took 26.222969ms I0429 06:57:10.959345 3947 pods.go:271] [openshift-operators/tekton-operator-webhook-5864bd5898-79wzz] addLogicalPort took 27.463557ms, libovsdb time 769.655µs I0429 06:57:11.207836 3947 cni.go:452] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-b8cq9 2db42c938ce4b3c08fcb7275b1952332516e48e4feb78c893d717141fe86b866 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-b8cq9 2db42c938ce4b3c08fcb7275b1952332516e48e4feb78c893d717141fe86b866 network default NAD default NAD key ] I0429 06:57:11.217678 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: openshift-pipelines-operator-6bdf6c8794-b8cq9, hostIfaceName: 2db42c938ce4b3c, network: default, NAD default, SandboxID: "2db42c938ce4b3c08fcb7275b1952332516e48e4feb78c893d717141fe86b866", PCI device ID: , UID: "0bb0d211-779e-48e5-ae03-cf3579593f21", MAC: 0a:58:0a:81:00:1b, IPs: [10.129.0.27/23] I0429 06:57:11.269551 3947 cni.go:473] [openshift-operators/openshift-pipelines-operator-6bdf6c8794-b8cq9 2db42c938ce4b3c08fcb7275b1952332516e48e4feb78c893d717141fe86b866 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/openshift-pipelines-operator-6bdf6c8794-b8cq9 2db42c938ce4b3c08fcb7275b1952332516e48e4feb78c893d717141fe86b866 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ce:71:58:74:be:1e\",\"name\":\"2db42c938ce4b3c\"},{\"mac\":\"0a:58:0a:81:00:1b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/d054a494-eb42-4039-bf60-3ea879826081\"}],\"ips\":[{\"address\":\"10.129.0.27/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:57:11.315474 3947 cni.go:452] [openshift-operators/tekton-operator-webhook-5864bd5898-79wzz 017ec6d8ffa212edf3a902b38b83e2d9d44ed6293dc93d34101161eb7d8388b5 network default NAD default NAD key ] ADD starting CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-79wzz 017ec6d8ffa212edf3a902b38b83e2d9d44ed6293dc93d34101161eb7d8388b5 network default NAD default NAD key ] I0429 06:57:11.327901 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-operators, podName: tekton-operator-webhook-5864bd5898-79wzz, hostIfaceName: 017ec6d8ffa212e, network: default, NAD default, SandboxID: "017ec6d8ffa212edf3a902b38b83e2d9d44ed6293dc93d34101161eb7d8388b5", PCI device ID: , UID: "2bdfc265-6a69-4683-9395-03157eb8236a", MAC: 0a:58:0a:81:00:1c, IPs: [10.129.0.28/23] I0429 06:57:11.376675 3947 cni.go:473] [openshift-operators/tekton-operator-webhook-5864bd5898-79wzz 017ec6d8ffa212edf3a902b38b83e2d9d44ed6293dc93d34101161eb7d8388b5 network default NAD default NAD key default] ADD finished CNI request [openshift-operators/tekton-operator-webhook-5864bd5898-79wzz 017ec6d8ffa212edf3a902b38b83e2d9d44ed6293dc93d34101161eb7d8388b5 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:59:5a:90:ab:d6\",\"name\":\"017ec6d8ffa212e\"},{\"mac\":\"0a:58:0a:81:00:1c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/e7c0fe48-fc40-4f3d-82a7-d04f49f953df\"}],\"ips\":[{\"address\":\"10.129.0.28/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:57:15.984720 3947 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs] creating logical port cert-manager_cert-manager-webhook-7dc76cbb94-qrqvs for pod on switch ip-10-0-2-36.ec2.internal I0429 06:57:15.984818 3947 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs I0429 06:57:16.042755 3947 kube.go:256] Updating pod cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs I0429 06:57:16.071546 3947 pod.go:62] [cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs] pod update took 86.751874ms I0429 06:57:16.071569 3947 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs] addLogicalPort annotation time took 86.779371ms I0429 06:57:16.072801 3947 pods.go:271] [cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs] addLogicalPort took 88.104403ms, libovsdb time 857.7µs I0429 06:57:16.325693 3947 cni.go:452] [cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs b3a01f11dc13c75af42a3f4ee1df709df2259620615c89b57c88f61c66b10b7c network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs b3a01f11dc13c75af42a3f4ee1df709df2259620615c89b57c88f61c66b10b7c network default NAD default NAD key ] I0429 06:57:16.335604 3947 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-webhook-7dc76cbb94-qrqvs, hostIfaceName: b3a01f11dc13c75, network: default, NAD default, SandboxID: "b3a01f11dc13c75af42a3f4ee1df709df2259620615c89b57c88f61c66b10b7c", PCI device ID: , UID: "5c605bfa-4297-4f19-b9ff-e6a9a1ae5b72", MAC: 0a:58:0a:81:00:1d, IPs: [10.129.0.29/23] I0429 06:57:16.386471 3947 cni.go:473] [cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs b3a01f11dc13c75af42a3f4ee1df709df2259620615c89b57c88f61c66b10b7c network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-webhook-7dc76cbb94-qrqvs b3a01f11dc13c75af42a3f4ee1df709df2259620615c89b57c88f61c66b10b7c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"4e:35:ee:f6:0d:f2\",\"name\":\"b3a01f11dc13c75\"},{\"mac\":\"0a:58:0a:81:00:1d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/2e042234-2b6c-473d-8d71-9d310273c516\"}],\"ips\":[{\"address\":\"10.129.0.29/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:57:17.577873 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0429 06:57:17.595102 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0429 06:57:17.608620 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0429 06:57:17.629455 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service I0429 06:57:18.532638 3947 namespace.go:98] [trusted-artifact-signer] adding namespace I0429 06:57:18.533747 3947 namespace.go:102] [trusted-artifact-signer] adding namespace took 1.083623ms I0429 06:57:18.577734 3947 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 06:57:18.621645 3947 namespace.go:142] [trusted-artifact-signer] updating namespace W0429 06:57:18.748827 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service openshift-operators/rhtas-operator-controller-manager-metrics-service W0429 06:57:19.660720 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service konflux-operator/konflux-operator-controller-manager-metrics-service W0429 06:57:19.723690 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service cert-manager/cert-manager-webhook W0429 06:57:19.723712 3947 util.go:870] Failed to build global endpoints for port TCP/metrics: empty IP address endpoints for service cert-manager/cert-manager-webhook W0429 06:57:25.685515 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 10250: empty IP address endpoints for service cert-manager/cert-manager-webhook W0429 06:57:25.685542 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9402: empty IP address endpoints for service cert-manager/cert-manager-webhook I0429 06:57:25.747510 3947 namespace.go:98] [openshift-pipelines] adding namespace I0429 06:57:25.748495 3947 namespace.go:102] [openshift-pipelines] adding namespace took 961.294µs I0429 06:57:25.782518 3947 namespace.go:142] [openshift-pipelines] updating namespace I0429 06:57:26.594045 3947 base_network_controller_pods.go:487] [default/cert-manager/cert-manager-59c77684cc-fwrxd] creating logical port cert-manager_cert-manager-59c77684cc-fwrxd for pod on switch ip-10-0-2-36.ec2.internal I0429 06:57:26.594142 3947 kube.go:256] Updating pod cert-manager/cert-manager-59c77684cc-fwrxd I0429 06:57:26.613371 3947 pod.go:62] [cert-manager/cert-manager-59c77684cc-fwrxd] pod update took 19.251482ms I0429 06:57:26.613403 3947 base_network_controller_pods.go:951] [default/cert-manager/cert-manager-59c77684cc-fwrxd] addLogicalPort annotation time took 19.289101ms I0429 06:57:26.614823 3947 pods.go:271] [cert-manager/cert-manager-59c77684cc-fwrxd] addLogicalPort took 20.795182ms, libovsdb time 912.887µs I0429 06:57:26.615567 3947 namespace.go:142] [cert-manager] updating namespace I0429 06:57:26.947430 3947 namespace.go:142] [cert-manager-operator] updating namespace I0429 06:57:26.965846 3947 cni.go:452] [cert-manager/cert-manager-59c77684cc-fwrxd 9faf366cbac371469622bfe919bed7e6f19a2592594c969958fe66b1932a847c network default NAD default NAD key ] ADD starting CNI request [cert-manager/cert-manager-59c77684cc-fwrxd 9faf366cbac371469622bfe919bed7e6f19a2592594c969958fe66b1932a847c network default NAD default NAD key ] I0429 06:57:26.975804 3947 helper_linux.go:556] ConfigureOVS: namespace: cert-manager, podName: cert-manager-59c77684cc-fwrxd, hostIfaceName: 9faf366cbac3714, network: default, NAD default, SandboxID: "9faf366cbac371469622bfe919bed7e6f19a2592594c969958fe66b1932a847c", PCI device ID: , UID: "3b470ee5-82e5-4581-b3ad-e702ea67fa4a", MAC: 0a:58:0a:81:00:1e, IPs: [10.129.0.30/23] I0429 06:57:27.028143 3947 cni.go:473] [cert-manager/cert-manager-59c77684cc-fwrxd 9faf366cbac371469622bfe919bed7e6f19a2592594c969958fe66b1932a847c network default NAD default NAD key default] ADD finished CNI request [cert-manager/cert-manager-59c77684cc-fwrxd 9faf366cbac371469622bfe919bed7e6f19a2592594c969958fe66b1932a847c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"86:c1:2f:29:8b:87\",\"name\":\"9faf366cbac3714\"},{\"mac\":\"0a:58:0a:81:00:1e\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0b2a52ea-9bc5-4f80-bc27-c152a0ebe0d1\"}],\"ips\":[{\"address\":\"10.129.0.30/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:57:27.479126 3947 namespace.go:142] [konflux-operator] updating namespace I0429 06:57:27.689146 3947 namespace.go:142] [konflux-ui] updating namespace I0429 06:57:27.873297 3947 namespace.go:142] [open-cluster-management-2pvlgkdgsf526nqovfd11k6703l08h7f] updating namespace I0429 06:57:28.276460 3947 namespace.go:142] [rhbk-operator] updating namespace I0429 06:57:28.475944 3947 namespace.go:142] [rhtpa-operator] updating namespace I0429 06:57:28.673933 3947 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 06:57:28.872724 3947 namespace.go:142] [tsf] updating namespace I0429 06:57:29.087688 3947 namespace.go:142] [tsf-keycloak] updating namespace I0429 06:57:29.315015 3947 namespace.go:142] [tsf-tas] updating namespace I0429 06:57:29.555341 3947 namespace.go:142] [tsf-tpa] updating namespace I0429 06:57:29.792521 3947 namespace.go:142] [cert-manager] updating namespace I0429 06:57:29.811944 3947 namespace.go:142] [cert-manager-operator] updating namespace I0429 06:57:29.846954 3947 namespace.go:142] [default] updating namespace I0429 06:57:29.865961 3947 namespace.go:142] [konflux-operator] updating namespace I0429 06:57:29.888350 3947 namespace.go:142] [konflux-ui] updating namespace I0429 06:57:29.910703 3947 namespace.go:142] [open-cluster-management-2pvlgkdgsf526nqovfd11k6703l08h7f] updating namespace I0429 06:57:29.932691 3947 namespace.go:142] [openshift] updating namespace I0429 06:57:29.961749 3947 namespace.go:142] [rhbk-operator] updating namespace I0429 06:57:29.978530 3947 namespace.go:142] [rhtpa-operator] updating namespace I0429 06:57:29.991923 3947 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 06:57:30.009650 3947 namespace.go:142] [tsf] updating namespace I0429 06:57:30.060290 3947 namespace.go:142] [tsf-keycloak] updating namespace I0429 06:57:30.084024 3947 namespace.go:142] [tsf-tas] updating namespace I0429 06:57:30.122283 3947 namespace.go:142] [tsf-tpa] updating namespace I0429 06:57:30.296608 3947 namespace.go:142] [cert-manager] updating namespace I0429 06:57:30.420970 3947 namespace.go:142] [cert-manager-operator] updating namespace I0429 06:57:30.628910 3947 namespace.go:142] [default] updating namespace I0429 06:57:30.844609 3947 namespace.go:142] [konflux-operator] updating namespace I0429 06:57:30.977157 3947 namespace.go:142] [konflux-ui] updating namespace I0429 06:57:31.157210 3947 namespace.go:142] [open-cluster-management-2pvlgkdgsf526nqovfd11k6703l08h7f] updating namespace I0429 06:57:31.241246 3947 namespace.go:142] [openshift] updating namespace I0429 06:57:31.311675 3947 namespace.go:142] [rhbk-operator] updating namespace I0429 06:57:31.398683 3947 namespace.go:142] [rhtpa-operator] updating namespace I0429 06:57:31.485331 3947 namespace.go:142] [trusted-artifact-signer] updating namespace I0429 06:57:31.586645 3947 namespace.go:142] [tsf] updating namespace I0429 06:57:31.643011 3947 namespace.go:142] [tsf-keycloak] updating namespace I0429 06:57:31.731393 3947 namespace.go:142] [tsf-tas] updating namespace I0429 06:57:31.915706 3947 namespace.go:142] [tsf-tpa] updating namespace I0429 06:57:32.538884 3947 namespace.go:142] [openshift-pipelines] updating namespace I0429 06:57:40.642108 3947 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-6ff72] creating logical port openshift-pipelines_tekton-pipelines-webhook-7b6d6f4c94-6ff72 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:57:40.642236 3947 kube.go:256] Updating pod openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-6ff72 I0429 06:57:40.664411 3947 pod.go:62] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-6ff72] pod update took 22.199283ms I0429 06:57:40.664436 3947 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-6ff72] addLogicalPort annotation time took 22.233493ms I0429 06:57:40.665752 3947 pods.go:271] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-6ff72] addLogicalPort took 23.667984ms, libovsdb time 907.508µs I0429 06:57:40.855641 3947 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:57:40.856187 3947 obj_retry.go:498] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:57:40.890067 3947 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:57:40.890092 3947 obj_retry.go:492] Detected object tsf/test-tsf-subscriptions of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:57:40.895951 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-subscriptions I0429 06:57:40.895975 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-subscriptions I0429 06:57:41.010883 3947 cni.go:452] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-6ff72 18a13ce8421059eca75ab8925d71aea28079df49c47cf83bd3559e9fceb226eb network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-6ff72 18a13ce8421059eca75ab8925d71aea28079df49c47cf83bd3559e9fceb226eb network default NAD default NAD key ] I0429 06:57:41.021633 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-pipelines-webhook-7b6d6f4c94-6ff72, hostIfaceName: 18a13ce8421059e, network: default, NAD default, SandboxID: "18a13ce8421059eca75ab8925d71aea28079df49c47cf83bd3559e9fceb226eb", PCI device ID: , UID: "13402bf7-fed4-4019-9fc5-a60f122e5731", MAC: 0a:58:0a:81:00:1f, IPs: [10.129.0.31/23] I0429 06:57:41.076932 3947 cni.go:473] [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-6ff72 18a13ce8421059eca75ab8925d71aea28079df49c47cf83bd3559e9fceb226eb network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-pipelines-webhook-7b6d6f4c94-6ff72 18a13ce8421059eca75ab8925d71aea28079df49c47cf83bd3559e9fceb226eb network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9e:96:52:fc:31:ce\",\"name\":\"18a13ce8421059e\"},{\"mac\":\"0a:58:0a:81:00:1f\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/fb2cfc4d-64d5-4cf7-b392-352e27597a6f\"}],\"ips\":[{\"address\":\"10.129.0.31/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:57:43.803297 3947 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 06:57:43.803316 3947 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 06:57:43.803321 3947 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 06:57:43.803326 3947 util.go:870] Failed to build global endpoints for port TCP/https-webhook: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 06:57:44.903705 3947 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 06:57:44.903727 3947 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 06:57:44.903735 3947 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-events-controller W0429 06:57:44.967566 3947 util.go:870] Failed to build global endpoints for port TCP/probes: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 06:57:44.967605 3947 util.go:870] Failed to build global endpoints for port TCP/http-profiling: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller W0429 06:57:44.967611 3947 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-controller I0429 06:57:46.170488 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=11 I0429 06:57:53.188452 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.ClusterUserDefinedNetwork" totalItems=11 W0429 06:57:54.807382 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 06:57:54.807422 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 06:57:54.807429 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8008: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 06:57:54.807434 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-pipelines-webhook W0429 06:57:55.811670 3947 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-tpa/tpa-pgsql W0429 06:57:55.850643 3947 util.go:870] Failed to build global endpoints for port TCP/data: empty IP address endpoints for service tsf-keycloak/keycloak-pgsql I0429 06:57:56.159572 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressfirewall/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressFirewall" totalItems=11 I0429 06:58:06.506791 3947 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-k9dlr] creating logical port openshift-pipelines_tekton-triggers-webhook-6b5b45569f-k9dlr for pod on switch ip-10-0-2-36.ec2.internal I0429 06:58:06.506887 3947 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-webhook-6b5b45569f-k9dlr I0429 06:58:06.528415 3947 pod.go:62] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-k9dlr] pod update took 21.552147ms I0429 06:58:06.528440 3947 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-webhook-6b5b45569f-k9dlr] addLogicalPort annotation time took 21.585826ms I0429 06:58:06.529872 3947 pods.go:271] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-k9dlr] addLogicalPort took 23.102294ms, libovsdb time 1.038393ms I0429 06:58:06.688389 3947 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-ft842] creating logical port openshift-pipelines_tekton-triggers-core-interceptors-76f587ddc9-ft842 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:58:06.688481 3947 kube.go:256] Updating pod openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-ft842 I0429 06:58:06.726446 3947 pod.go:62] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-ft842] pod update took 37.987966ms I0429 06:58:06.726470 3947 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-ft842] addLogicalPort annotation time took 38.019437ms I0429 06:58:06.727677 3947 pods.go:271] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-ft842] addLogicalPort took 39.303713ms, libovsdb time 851.098µs I0429 06:58:06.867236 3947 cni.go:452] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-k9dlr 9bf54b8c9faaea687bc797f17b945ad22cbeb14e49aefb329987fbe33aed71b6 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-k9dlr 9bf54b8c9faaea687bc797f17b945ad22cbeb14e49aefb329987fbe33aed71b6 network default NAD default NAD key ] I0429 06:58:06.876943 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-webhook-6b5b45569f-k9dlr, hostIfaceName: 9bf54b8c9faaea6, network: default, NAD default, SandboxID: "9bf54b8c9faaea687bc797f17b945ad22cbeb14e49aefb329987fbe33aed71b6", PCI device ID: , UID: "76577b1c-0656-428c-83f0-c7889e152d47", MAC: 0a:58:0a:81:00:20, IPs: [10.129.0.32/23] I0429 06:58:06.930179 3947 cni.go:473] [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-k9dlr 9bf54b8c9faaea687bc797f17b945ad22cbeb14e49aefb329987fbe33aed71b6 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-webhook-6b5b45569f-k9dlr 9bf54b8c9faaea687bc797f17b945ad22cbeb14e49aefb329987fbe33aed71b6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"1a:1e:b6:48:35:b6\",\"name\":\"9bf54b8c9faaea6\"},{\"mac\":\"0a:58:0a:81:00:20\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a8bbf7d6-f0f0-4801-aaa7-1077e65c7546\"}],\"ips\":[{\"address\":\"10.129.0.32/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:58:07.045097 3947 cni.go:452] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-ft842 476cb7ecfec01ae0afa56f0718acc0c0b0cbdc55a34db7f1b17aa7d0993bdcc1 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-ft842 476cb7ecfec01ae0afa56f0718acc0c0b0cbdc55a34db7f1b17aa7d0993bdcc1 network default NAD default NAD key ] I0429 06:58:07.055308 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-triggers-core-interceptors-76f587ddc9-ft842, hostIfaceName: 476cb7ecfec01ae, network: default, NAD default, SandboxID: "476cb7ecfec01ae0afa56f0718acc0c0b0cbdc55a34db7f1b17aa7d0993bdcc1", PCI device ID: , UID: "a574f2d6-ceb0-47b7-b048-a6b3749e497d", MAC: 0a:58:0a:81:00:21, IPs: [10.129.0.33/23] I0429 06:58:07.107517 3947 cni.go:473] [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-ft842 476cb7ecfec01ae0afa56f0718acc0c0b0cbdc55a34db7f1b17aa7d0993bdcc1 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-triggers-core-interceptors-76f587ddc9-ft842 476cb7ecfec01ae0afa56f0718acc0c0b0cbdc55a34db7f1b17aa7d0993bdcc1 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ba:ae:10:2c:aa:35\",\"name\":\"476cb7ecfec01ae\"},{\"mac\":\"0a:58:0a:81:00:21\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/84bcd87c-64c0-4a0c-be55-bbdbb705d011\"}],\"ips\":[{\"address\":\"10.129.0.33/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:58:10.188175 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1/apis/informers/externalversions/factory.go:140" type="*v1.UserDefinedNetwork" totalItems=11 W0429 06:58:10.964932 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors I0429 06:58:19.050782 3947 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:58:19.051340 3947 obj_retry.go:498] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:58:19.072578 3947 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:58:19.072622 3947 obj_retry.go:492] Detected object tsf/test-tsf-infrastructure of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:58:19.081919 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-infrastructure I0429 06:58:19.081945 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-infrastructure I0429 06:58:21.105718 3947 base_network_controller_policy.go:1179] Adding network policy tsf-keycloak/keycloak-network-policy for network default I0429 06:58:21.106397 3947 pod_selector_address_set.go:213] Created shared address set for pod selector tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},} I0429 06:58:21.106420 3947 base_network_controller_policy.go:1054] Policy tsf-keycloak/keycloak-network-policy added to peer address sets [tsf-keycloak_LS{ML:{app: keycloak,app.kubernetes.io/instance: keycloak,app.kubernetes.io/managed-by: keycloak-operator,},}] I0429 06:58:21.108704 3947 base_network_controller_policy.go:1232] Create network policy tsf-keycloak/keycloak-network-policy resources completed, update namespace loglevel W0429 06:58:21.937337 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8443: empty IP address endpoints for service openshift-pipelines/tekton-triggers-core-interceptors W0429 06:58:27.978213 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-keycloak/keycloak W0429 06:58:27.978232 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak W0429 06:58:27.983172 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service tsf-keycloak/keycloak-service W0429 06:58:27.983188 3947 util.go:870] Failed to build global endpoints for port TCP/management: empty IP address endpoints for service tsf-keycloak/keycloak-service I0429 06:58:28.090334 3947 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0429 06:58:31.182914 3947 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-postgres-0] creating logical port openshift-pipelines_tekton-results-postgres-0 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:58:31.183012 3947 kube.go:256] Updating pod openshift-pipelines/tekton-results-postgres-0 I0429 06:58:31.199522 3947 pod.go:62] [openshift-pipelines/tekton-results-postgres-0] pod update took 16.537289ms I0429 06:58:31.199547 3947 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-postgres-0] addLogicalPort annotation time took 16.565675ms I0429 06:58:31.200802 3947 pods.go:271] [openshift-pipelines/tekton-results-postgres-0] addLogicalPort took 17.909197ms, libovsdb time 886.106µs I0429 06:58:35.446798 3947 cni.go:452] [openshift-pipelines/tekton-results-postgres-0 39f69ed45d11761f67107d3625037fad1e951bceb426f97d760da36837c0e367 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-postgres-0 39f69ed45d11761f67107d3625037fad1e951bceb426f97d760da36837c0e367 network default NAD default NAD key ] I0429 06:58:35.456195 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-postgres-0, hostIfaceName: 39f69ed45d11761, network: default, NAD default, SandboxID: "39f69ed45d11761f67107d3625037fad1e951bceb426f97d760da36837c0e367", PCI device ID: , UID: "d6bb0c5a-3e0c-432a-a473-34a1c9268da0", MAC: 0a:58:0a:81:00:22, IPs: [10.129.0.34/23] I0429 06:58:35.508750 3947 cni.go:473] [openshift-pipelines/tekton-results-postgres-0 39f69ed45d11761f67107d3625037fad1e951bceb426f97d760da36837c0e367 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-postgres-0 39f69ed45d11761f67107d3625037fad1e951bceb426f97d760da36837c0e367 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"ea:cf:5b:85:70:b7\",\"name\":\"39f69ed45d11761\"},{\"mac\":\"0a:58:0a:81:00:22\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/21428226-1fcb-483f-8b85-a36e776419ed\"}],\"ips\":[{\"address\":\"10.129.0.34/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:58:41.122508 3947 iptables.go:108] Creating table: nat chain: OVN-KUBE-NODEPORT I0429 06:58:42.492851 3947 base_network_controller_pods.go:487] [default/openshift-pipelines/tekton-results-api-64c5c8474c-rs5s7] creating logical port openshift-pipelines_tekton-results-api-64c5c8474c-rs5s7 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:58:42.492966 3947 kube.go:256] Updating pod openshift-pipelines/tekton-results-api-64c5c8474c-rs5s7 I0429 06:58:42.507697 3947 pod.go:62] [openshift-pipelines/tekton-results-api-64c5c8474c-rs5s7] pod update took 14.771807ms I0429 06:58:42.507719 3947 base_network_controller_pods.go:951] [default/openshift-pipelines/tekton-results-api-64c5c8474c-rs5s7] addLogicalPort annotation time took 14.799662ms I0429 06:58:42.508967 3947 pods.go:271] [openshift-pipelines/tekton-results-api-64c5c8474c-rs5s7] addLogicalPort took 16.130919ms, libovsdb time 875.34µs I0429 06:58:42.854855 3947 cni.go:452] [openshift-pipelines/tekton-results-api-64c5c8474c-rs5s7 62af971f8f1154705b96f5c8d317bdf66ecdf2d282550618727d2474d14a731c network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-rs5s7 62af971f8f1154705b96f5c8d317bdf66ecdf2d282550618727d2474d14a731c network default NAD default NAD key ] I0429 06:58:42.864650 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: tekton-results-api-64c5c8474c-rs5s7, hostIfaceName: 62af971f8f11547, network: default, NAD default, SandboxID: "62af971f8f1154705b96f5c8d317bdf66ecdf2d282550618727d2474d14a731c", PCI device ID: , UID: "43922ada-ed11-4788-a6fa-3a71b253795b", MAC: 0a:58:0a:81:00:23, IPs: [10.129.0.35/23] I0429 06:58:42.918941 3947 cni.go:473] [openshift-pipelines/tekton-results-api-64c5c8474c-rs5s7 62af971f8f1154705b96f5c8d317bdf66ecdf2d282550618727d2474d14a731c network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/tekton-results-api-64c5c8474c-rs5s7 62af971f8f1154705b96f5c8d317bdf66ecdf2d282550618727d2474d14a731c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"e2:73:ca:86:86:d9\",\"name\":\"62af971f8f11547\"},{\"mac\":\"0a:58:0a:81:00:23\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/7d17de15-75c7-4f44-9fdd-a0d0925b52bd\"}],\"ips\":[{\"address\":\"10.129.0.35/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:58:47.127601 3947 util.go:870] Failed to build global endpoints for port TCP/prometheus: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0429 06:58:47.127617 3947 util.go:870] Failed to build global endpoints for port TCP/profiling: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0429 06:58:47.127621 3947 util.go:870] Failed to build global endpoints for port TCP/server: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0429 06:58:52.262762 3947 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:58:52.263286 3947 obj_retry.go:498] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:58:52.307985 3947 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:58:52.308011 3947 obj_retry.go:492] Detected object tsf/test-tsf-iam of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:58:52.317227 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/test-tsf-iam I0429 06:58:52.317247 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/test-tsf-iam W0429 06:58:53.148070 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0429 06:58:53.148089 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 6060: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service W0429 06:58:53.148094 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service openshift-pipelines/tekton-results-api-service I0429 06:58:55.093989 3947 base_network_controller_pods.go:487] [default/tsf-tas/fulcio-server-5cdc57c897-l9ks9] creating logical port tsf-tas_fulcio-server-5cdc57c897-l9ks9 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:58:55.094075 3947 kube.go:256] Updating pod tsf-tas/fulcio-server-5cdc57c897-l9ks9 I0429 06:58:55.108168 3947 pod.go:62] [tsf-tas/fulcio-server-5cdc57c897-l9ks9] pod update took 14.111613ms I0429 06:58:55.108194 3947 base_network_controller_pods.go:951] [default/tsf-tas/fulcio-server-5cdc57c897-l9ks9] addLogicalPort annotation time took 14.142094ms I0429 06:58:55.109757 3947 pods.go:271] [tsf-tas/fulcio-server-5cdc57c897-l9ks9] addLogicalPort took 15.783641ms, libovsdb time 958.096µs I0429 06:58:56.965004 3947 cni.go:452] [tsf-tas/fulcio-server-5cdc57c897-l9ks9 2924ee10458f414aebf0173506f4fa29b2ad4d3ccf816caad54ab798acc6705f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/fulcio-server-5cdc57c897-l9ks9 2924ee10458f414aebf0173506f4fa29b2ad4d3ccf816caad54ab798acc6705f network default NAD default NAD key ] I0429 06:58:56.991305 3947 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: fulcio-server-5cdc57c897-l9ks9, hostIfaceName: 2924ee10458f414, network: default, NAD default, SandboxID: "2924ee10458f414aebf0173506f4fa29b2ad4d3ccf816caad54ab798acc6705f", PCI device ID: , UID: "fbd8fff4-32ba-4007-b3ae-6a51a1305627", MAC: 0a:58:0a:81:00:24, IPs: [10.129.0.36/23] I0429 06:58:57.043002 3947 cni.go:473] [tsf-tas/fulcio-server-5cdc57c897-l9ks9 2924ee10458f414aebf0173506f4fa29b2ad4d3ccf816caad54ab798acc6705f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/fulcio-server-5cdc57c897-l9ks9 2924ee10458f414aebf0173506f4fa29b2ad4d3ccf816caad54ab798acc6705f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"9a:eb:13:38:1a:fa\",\"name\":\"2924ee10458f414\"},{\"mac\":\"0a:58:0a:81:00:24\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/af3092fb-fd84-4fa5-ba19-eaccdc09e2ec\"}],\"ips\":[{\"address\":\"10.129.0.36/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:58:58.720029 3947 base_network_controller_pods.go:487] [default/tsf-tas/trillian-db-64946c9d67-n8sr7] creating logical port tsf-tas_trillian-db-64946c9d67-n8sr7 for pod on switch ip-10-0-2-36.ec2.internal I0429 06:58:58.720135 3947 kube.go:256] Updating pod tsf-tas/trillian-db-64946c9d67-n8sr7 I0429 06:58:58.735545 3947 pod.go:62] [tsf-tas/trillian-db-64946c9d67-n8sr7] pod update took 15.443974ms I0429 06:58:58.735571 3947 base_network_controller_pods.go:951] [default/tsf-tas/trillian-db-64946c9d67-n8sr7] addLogicalPort annotation time took 15.47539ms I0429 06:58:58.736939 3947 pods.go:271] [tsf-tas/trillian-db-64946c9d67-n8sr7] addLogicalPort took 16.92715ms, libovsdb time 961.471µs I0429 06:59:05.379737 3947 cni.go:452] [tsf-tas/trillian-db-64946c9d67-n8sr7 6910cfdffe4cdda8fbb8a25184314973116052a6ac9372216a9ba4f943ca622f network default NAD default NAD key ] ADD starting CNI request [tsf-tas/trillian-db-64946c9d67-n8sr7 6910cfdffe4cdda8fbb8a25184314973116052a6ac9372216a9ba4f943ca622f network default NAD default NAD key ] I0429 06:59:05.389035 3947 helper_linux.go:556] ConfigureOVS: namespace: tsf-tas, podName: trillian-db-64946c9d67-n8sr7, hostIfaceName: 6910cfdffe4cdda, network: default, NAD default, SandboxID: "6910cfdffe4cdda8fbb8a25184314973116052a6ac9372216a9ba4f943ca622f", PCI device ID: , UID: "37b6bb4e-cf5d-467f-931d-f444f0b714e1", MAC: 0a:58:0a:81:00:25, IPs: [10.129.0.37/23] I0429 06:59:05.440816 3947 cni.go:473] [tsf-tas/trillian-db-64946c9d67-n8sr7 6910cfdffe4cdda8fbb8a25184314973116052a6ac9372216a9ba4f943ca622f network default NAD default NAD key default] ADD finished CNI request [tsf-tas/trillian-db-64946c9d67-n8sr7 6910cfdffe4cdda8fbb8a25184314973116052a6ac9372216a9ba4f943ca622f network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5e:b0:2a:f8:bd:32\",\"name\":\"6910cfdffe4cdda\"},{\"mac\":\"0a:58:0a:81:00:25\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/8fecd270-8286-48ad-971c-413f80f097dc\"}],\"ips\":[{\"address\":\"10.129.0.37/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 06:59:13.249860 3947 util.go:870] Failed to build global endpoints for port TCP/trillian-mysql: empty IP address endpoints for service tsf-tas/trillian-mysql W0429 06:59:14.265254 3947 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/fulcio-server W0429 06:59:14.265274 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/fulcio-server W0429 06:59:14.300501 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5554: empty IP address endpoints for service tsf-tas/fulcio-server W0429 06:59:14.300524 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 5555: empty IP address endpoints for service tsf-tas/fulcio-server I0429 06:59:18.041267 3947 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ctrzl] creating logical port openshift-pipelines_pipelines-as-code-watcher-7947cf58bc-ctrzl for pod on switch ip-10-0-2-36.ec2.internal I0429 06:59:18.041352 3947 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ctrzl I0429 06:59:18.055034 3947 pod.go:62] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ctrzl] pod update took 13.702975ms I0429 06:59:18.055055 3947 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ctrzl] addLogicalPort annotation time took 13.728033ms I0429 06:59:18.056645 3947 pods.go:271] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ctrzl] addLogicalPort took 15.391561ms, libovsdb time 1.154507ms I0429 06:59:18.141766 3947 base_network_controller_pods.go:487] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-zm5tr] creating logical port openshift-pipelines_pipelines-as-code-webhook-68965d5857-zm5tr for pod on switch ip-10-0-2-36.ec2.internal I0429 06:59:18.141889 3947 kube.go:256] Updating pod openshift-pipelines/pipelines-as-code-webhook-68965d5857-zm5tr I0429 06:59:18.156499 3947 pod.go:62] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-zm5tr] pod update took 14.631226ms I0429 06:59:18.156520 3947 base_network_controller_pods.go:951] [default/openshift-pipelines/pipelines-as-code-webhook-68965d5857-zm5tr] addLogicalPort annotation time took 14.655534ms I0429 06:59:18.157786 3947 pods.go:271] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-zm5tr] addLogicalPort took 16.059973ms, libovsdb time 843.118µs I0429 06:59:18.204292 3947 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-bgnjh of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:59:18.204827 3947 obj_retry.go:498] Detected object tsf-keycloak/tsf-iam-bgnjh of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0429 06:59:18.371392 3947 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logsigner W0429 06:59:18.399289 3947 util.go:870] Failed to build global endpoints for port TCP/grpc: empty IP address endpoints for service tsf-tas/trillian-logserver I0429 06:59:18.420912 3947 cni.go:452] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ctrzl 90c4623d561c01991f4a712a22ca84b784dbafbe3004fecaa5ce9967e159dae6 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ctrzl 90c4623d561c01991f4a712a22ca84b784dbafbe3004fecaa5ce9967e159dae6 network default NAD default NAD key ] I0429 06:59:18.430469 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-watcher-7947cf58bc-ctrzl, hostIfaceName: 90c4623d561c019, network: default, NAD default, SandboxID: "90c4623d561c01991f4a712a22ca84b784dbafbe3004fecaa5ce9967e159dae6", PCI device ID: , UID: "61db4040-f2f0-4dd9-8ebc-ab0c70ecc7bc", MAC: 0a:58:0a:81:00:26, IPs: [10.129.0.38/23] I0429 06:59:18.482256 3947 cni.go:473] [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ctrzl 90c4623d561c01991f4a712a22ca84b784dbafbe3004fecaa5ce9967e159dae6 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-watcher-7947cf58bc-ctrzl 90c4623d561c01991f4a712a22ca84b784dbafbe3004fecaa5ce9967e159dae6 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"3e:84:65:ce:ed:5b\",\"name\":\"90c4623d561c019\"},{\"mac\":\"0a:58:0a:81:00:26\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/0f115856-c95f-4b58-b5c8-54a041cbeee3\"}],\"ips\":[{\"address\":\"10.129.0.38/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:59:18.534496 3947 cni.go:452] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-zm5tr 5fbaefe383cab9bef6fee843611c2ebc170899022d98c4728460cc7478d14386 network default NAD default NAD key ] ADD starting CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-zm5tr 5fbaefe383cab9bef6fee843611c2ebc170899022d98c4728460cc7478d14386 network default NAD default NAD key ] I0429 06:59:18.544668 3947 helper_linux.go:556] ConfigureOVS: namespace: openshift-pipelines, podName: pipelines-as-code-webhook-68965d5857-zm5tr, hostIfaceName: 5fbaefe383cab9b, network: default, NAD default, SandboxID: "5fbaefe383cab9bef6fee843611c2ebc170899022d98c4728460cc7478d14386", PCI device ID: , UID: "0a3e93d5-4df3-4195-b46e-baa10f1686ac", MAC: 0a:58:0a:81:00:27, IPs: [10.129.0.39/23] I0429 06:59:18.595625 3947 cni.go:473] [openshift-pipelines/pipelines-as-code-webhook-68965d5857-zm5tr 5fbaefe383cab9bef6fee843611c2ebc170899022d98c4728460cc7478d14386 network default NAD default NAD key default] ADD finished CNI request [openshift-pipelines/pipelines-as-code-webhook-68965d5857-zm5tr 5fbaefe383cab9bef6fee843611c2ebc170899022d98c4728460cc7478d14386 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"5a:f4:6e:97:fe:36\",\"name\":\"5fbaefe383cab9b\"},{\"mac\":\"0a:58:0a:81:00:27\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/6b8899e4-3220-4e87-9b0d-13aa2667adac\"}],\"ips\":[{\"address\":\"10.129.0.39/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 06:59:19.234928 3947 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-bgnjh of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:59:19.234954 3947 obj_retry.go:492] Detected object tsf-keycloak/tsf-iam-bgnjh of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 06:59:21.336188 3947 util.go:870] Failed to build global endpoints for port TCP/http-metrics: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher W0429 06:59:24.296847 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 3306: empty IP address endpoints for service tsf-tas/trillian-mysql I0429 06:59:24.481893 3947 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-schct-74tvg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:59:24.482429 3947 obj_retry.go:498] Detected object tsf-tas/segment-backup-installation-schct-74tvg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:59:25.352461 3947 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-schct-74tvg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:59:25.352482 3947 obj_retry.go:492] Detected object tsf-tas/segment-backup-installation-schct-74tvg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:59:30.067689 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Namespace" totalItems=96 W0429 06:59:32.298097 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9090: empty IP address endpoints for service openshift-pipelines/pipelines-as-code-watcher I0429 06:59:52.660926 3947 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 06:59:52.661498 3947 obj_retry.go:498] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 06:59:52.684097 3947 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:59:52.684123 3947 obj_retry.go:492] Detected object tsf-tas/test-tsf-tas of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 06:59:52.692644 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tas/test-tsf-tas I0429 06:59:52.692663 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tas/test-tsf-tas I0429 07:00:01.779120 3947 obj_retry.go:498] Detected object tsf/patch-tekton-config-9qqrs of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:01.779720 3947 obj_retry.go:498] Detected object tsf/patch-tekton-config-9qqrs of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:02.649241 3947 obj_retry.go:492] Detected object tsf/patch-tekton-config-9qqrs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:02.649271 3947 obj_retry.go:492] Detected object tsf/patch-tekton-config-9qqrs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:03.783123 3947 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624100-krpl4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:03.783694 3947 obj_retry.go:498] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624100-krpl4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:04.661924 3947 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624100-krpl4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:04.661954 3947 obj_retry.go:492] Detected object openshift-backplane/osd-delete-backplane-serviceaccounts-29624100-krpl4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:05.179903 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/factory.go:140" type="*v1.AdminPolicyBasedExternalRoute" totalItems=7 I0429 07:00:10.716796 3947 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-nbsht of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:10.717365 3947 obj_retry.go:498] Detected object tsf/tsf-tekton-configuration-nbsht of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:11.741925 3947 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-nbsht of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:11.741954 3947 obj_retry.go:492] Detected object tsf/tsf-tekton-configuration-nbsht of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:11.783978 3947 obj_retry.go:492] Detected object tsf/patch-tekton-config-9qqrs of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:11.784001 3947 obj_retry.go:492] Detected object tsf/patch-tekton-config-9qqrs of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:11.788535 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf/patch-tekton-config-9qqrs I0429 07:00:11.788556 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf/patch-tekton-config-9qqrs I0429 07:00:14.770155 3947 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-bv4xs-8s7p4 of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:14.770736 3947 obj_retry.go:498] Detected object tsf-tas/ctlog-createtree-job-bv4xs-8s7p4 of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:15.658655 3947 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-bv4xs-8s7p4 of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:15.658691 3947 obj_retry.go:492] Detected object tsf-tas/ctlog-createtree-job-bv4xs-8s7p4 of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:15.744978 3947 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:15.745541 3947 obj_retry.go:498] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:15.767153 3947 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:15.767173 3947 obj_retry.go:492] Detected object tsf-tpa/pre-install-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:15.780596 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/pre-install-tsf-tpa I0429 07:00:15.780620 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/pre-install-tsf-tpa I0429 07:00:19.865436 3947 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-vzxvc-ndhbv of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:19.866089 3947 obj_retry.go:498] Detected object tsf-tas/rekor-createtree-job-vzxvc-ndhbv of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0429 07:00:20.752665 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/ctlog I0429 07:00:20.756600 3947 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-vzxvc-ndhbv of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:20.756622 3947 obj_retry.go:492] Detected object tsf-tas/rekor-createtree-job-vzxvc-ndhbv of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:21.810306 3947 obj_retry.go:498] Detected object tsf-tpa/create-db-jdw4t of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:21.810890 3947 obj_retry.go:498] Detected object tsf-tpa/create-db-jdw4t of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:22.837578 3947 obj_retry.go:492] Detected object tsf-tpa/create-db-jdw4t of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:22.837623 3947 obj_retry.go:492] Detected object tsf-tpa/create-db-jdw4t of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 07:00:25.760891 3947 util.go:870] Failed to build global endpoints for port TCP/resp: empty IP address endpoints for service tsf-tas/rekor-redis I0429 07:00:26.533190 3947 obj_retry.go:498] Detected object tsf-tpa/migrate-db-6rmtn of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:26.533838 3947 obj_retry.go:498] Detected object tsf-tpa/migrate-db-6rmtn of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it W0429 07:00:26.713858 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-search-ui I0429 07:00:27.561993 3947 obj_retry.go:492] Detected object tsf-tpa/migrate-db-6rmtn of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:27.562013 3947 obj_retry.go:492] Detected object tsf-tpa/migrate-db-6rmtn of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 07:00:30.760684 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/rekor-server I0429 07:00:31.928849 3947 obj_retry.go:498] Detected object tsf-tpa/create-importers-4cpzm of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:31.929485 3947 obj_retry.go:498] Detected object tsf-tpa/create-importers-4cpzm of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:00:32.955266 3947 obj_retry.go:492] Detected object tsf-tpa/create-importers-4cpzm of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:00:32.955295 3947 obj_retry.go:492] Detected object tsf-tpa/create-importers-4cpzm of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 07:00:38.791360 3947 util.go:870] Failed to build global endpoints for port TCP/endpoint: empty IP address endpoints for service tsf-tpa/server I0429 07:01:05.063606 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.EndpointSlice" totalItems=114 I0429 07:01:07.063615 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.NetworkPolicy" totalItems=7 I0429 07:01:07.072764 3947 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:01:07.073420 3947 obj_retry.go:498] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:01:07.117514 3947 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:01:07.117533 3947 obj_retry.go:492] Detected object tsf-tpa/test-tsf-tpa of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:01:07.126298 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod tsf-tpa/test-tsf-tpa I0429 07:01:07.126324 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod tsf-tpa/test-tsf-tpa I0429 07:01:07.950114 3947 namespace.go:98] [build-service] adding namespace I0429 07:01:07.951085 3947 namespace.go:102] [build-service] adding namespace took 941.089µs I0429 07:01:07.951244 3947 namespace.go:98] [integration-service] adding namespace I0429 07:01:07.951939 3947 namespace.go:102] [integration-service] adding namespace took 673.828µs I0429 07:01:07.954410 3947 namespace.go:98] [default-tenant] adding namespace I0429 07:01:07.955094 3947 namespace.go:102] [default-tenant] adding namespace took 670.151µs I0429 07:01:07.957399 3947 namespace.go:98] [image-controller] adding namespace I0429 07:01:07.958276 3947 namespace.go:102] [image-controller] adding namespace took 856.266µs I0429 07:01:08.018725 3947 namespace.go:142] [build-service] updating namespace I0429 07:01:08.045056 3947 namespace.go:142] [build-service] updating namespace I0429 07:01:08.067003 3947 namespace.go:142] [integration-service] updating namespace I0429 07:01:08.093768 3947 namespace.go:142] [integration-service] updating namespace I0429 07:01:08.185453 3947 namespace.go:142] [build-service] updating namespace I0429 07:01:08.185451 3947 namespace.go:142] [default-tenant] updating namespace I0429 07:01:08.224944 3947 namespace.go:142] [integration-service] updating namespace I0429 07:01:08.230100 3947 namespace.go:142] [image-controller] updating namespace I0429 07:01:08.248212 3947 namespace.go:142] [default-tenant] updating namespace I0429 07:01:08.269959 3947 namespace.go:98] [release-service] adding namespace I0429 07:01:08.270951 3947 namespace.go:102] [release-service] adding namespace took 967.588µs I0429 07:01:08.284945 3947 namespace.go:142] [image-controller] updating namespace I0429 07:01:08.316602 3947 namespace.go:142] [konflux-ui] updating namespace I0429 07:01:08.325650 3947 namespace.go:142] [release-service] updating namespace I0429 07:01:08.372963 3947 namespace.go:142] [release-service] updating namespace I0429 07:01:08.387542 3947 base_network_controller_pods.go:487] [default/konflux-ui/test-tsf-konflux] creating logical port konflux-ui_test-tsf-konflux for pod on switch ip-10-0-2-36.ec2.internal I0429 07:01:08.387700 3947 kube.go:256] Updating pod konflux-ui/test-tsf-konflux I0429 07:01:08.403630 3947 namespace.go:98] [konflux-info] adding namespace I0429 07:01:08.404546 3947 namespace.go:102] [konflux-info] adding namespace took 890.738µs I0429 07:01:08.418313 3947 pod.go:62] [konflux-ui/test-tsf-konflux] pod update took 30.637594ms I0429 07:01:08.418337 3947 base_network_controller_pods.go:951] [default/konflux-ui/test-tsf-konflux] addLogicalPort annotation time took 30.669001ms I0429 07:01:08.419579 3947 pods.go:271] [konflux-ui/test-tsf-konflux] addLogicalPort took 32.053345ms, libovsdb time 812.459µs I0429 07:01:08.451606 3947 namespace.go:98] [namespace-lister] adding namespace I0429 07:01:08.452425 3947 namespace.go:102] [namespace-lister] adding namespace took 797.956µs I0429 07:01:08.500541 3947 namespace.go:98] [enterprise-contract-service] adding namespace I0429 07:01:08.501625 3947 namespace.go:102] [enterprise-contract-service] adding namespace took 1.057678ms I0429 07:01:08.538613 3947 namespace.go:142] [image-controller] updating namespace I0429 07:01:08.549889 3947 namespace.go:142] [konflux-info] updating namespace I0429 07:01:08.591478 3947 namespace.go:142] [konflux-info] updating namespace I0429 07:01:08.641244 3947 namespace.go:142] [namespace-lister] updating namespace I0429 07:01:08.708171 3947 namespace.go:142] [namespace-lister] updating namespace I0429 07:01:08.722068 3947 namespace.go:142] [enterprise-contract-service] updating namespace I0429 07:01:08.735569 3947 namespace.go:142] [default-tenant] updating namespace I0429 07:01:08.752047 3947 cni.go:452] [konflux-ui/test-tsf-konflux d84e18157ce3b97f4fd828acae9c54c6d6bd49cc39b363ff8460aaa0d7c5af10 network default NAD default NAD key ] ADD starting CNI request [konflux-ui/test-tsf-konflux d84e18157ce3b97f4fd828acae9c54c6d6bd49cc39b363ff8460aaa0d7c5af10 network default NAD default NAD key ] I0429 07:01:08.761656 3947 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: test-tsf-konflux, hostIfaceName: d84e18157ce3b97, network: default, NAD default, SandboxID: "d84e18157ce3b97f4fd828acae9c54c6d6bd49cc39b363ff8460aaa0d7c5af10", PCI device ID: , UID: "a4f422a3-35e1-4f4d-8902-c21f16c47cbf", MAC: 0a:58:0a:81:00:28, IPs: [10.129.0.40/23] I0429 07:01:08.825932 3947 cni.go:473] [konflux-ui/test-tsf-konflux d84e18157ce3b97f4fd828acae9c54c6d6bd49cc39b363ff8460aaa0d7c5af10 network default NAD default NAD key default] ADD finished CNI request [konflux-ui/test-tsf-konflux d84e18157ce3b97f4fd828acae9c54c6d6bd49cc39b363ff8460aaa0d7c5af10 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"f2:ed:1f:47:e0:91\",\"name\":\"d84e18157ce3b97\"},{\"mac\":\"0a:58:0a:81:00:28\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/65e289f7-ee49-44eb-8056-3857c9d8478b\"}],\"ips\":[{\"address\":\"10.129.0.40/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 07:01:08.845574 3947 namespace.go:98] [segment-bridge] adding namespace I0429 07:01:08.846369 3947 namespace.go:102] [segment-bridge] adding namespace took 759.282µs I0429 07:01:08.886568 3947 namespace.go:98] [konflux-cli] adding namespace I0429 07:01:08.887503 3947 namespace.go:102] [konflux-cli] adding namespace took 884.671µs I0429 07:01:08.973940 3947 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-from-konfluxui for network default I0429 07:01:08.974714 3947 pod_selector_address_set.go:213] Created shared address set for pod selector namespace-lister_LS{ML:{app: proxy,},} I0429 07:01:08.974737 3947 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-from-konfluxui added to peer address sets [namespace-lister_LS{ML:{app: proxy,},}] I0429 07:01:08.977218 3947 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-from-konfluxui resources completed, update namespace loglevel I0429 07:01:09.021668 3947 namespace.go:142] [enterprise-contract-service] updating namespace I0429 07:01:09.029469 3947 base_network_controller_policy.go:1179] Adding network policy namespace-lister/namespace-lister-allow-to-apiserver for network default I0429 07:01:09.029497 3947 base_network_controller_policy.go:1054] Policy namespace-lister/namespace-lister-allow-to-apiserver added to peer address sets [] I0429 07:01:09.030464 3947 base_network_controller_policy.go:1232] Create network policy namespace-lister/namespace-lister-allow-to-apiserver resources completed, update namespace loglevel I0429 07:01:09.593572 3947 namespace.go:142] [segment-bridge] updating namespace I0429 07:01:09.681892 3947 namespace.go:142] [segment-bridge] updating namespace I0429 07:01:09.689860 3947 base_network_controller_pods.go:487] [default/namespace-lister/namespace-lister-696b855848-5wm97] creating logical port namespace-lister_namespace-lister-696b855848-5wm97 for pod on switch ip-10-0-2-36.ec2.internal I0429 07:01:09.689949 3947 kube.go:256] Updating pod namespace-lister/namespace-lister-696b855848-5wm97 I0429 07:01:09.744725 3947 pod.go:62] [namespace-lister/namespace-lister-696b855848-5wm97] pod update took 54.798167ms I0429 07:01:09.744755 3947 base_network_controller_pods.go:951] [default/namespace-lister/namespace-lister-696b855848-5wm97] addLogicalPort annotation time took 54.834293ms I0429 07:01:09.746079 3947 pods.go:271] [namespace-lister/namespace-lister-696b855848-5wm97] addLogicalPort took 56.238928ms, libovsdb time 926.506µs I0429 07:01:09.748515 3947 namespace.go:142] [konflux-cli] updating namespace I0429 07:01:10.002645 3947 namespace.go:142] [konflux-cli] updating namespace I0429 07:01:11.032942 3947 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-7b666bf6d8-nllnc] creating logical port integration-service_integration-service-controller-manager-7b666bf6d8-nllnc for pod on switch ip-10-0-2-36.ec2.internal I0429 07:01:11.033074 3947 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7b666bf6d8-nllnc I0429 07:01:11.133438 3947 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7b666bf6d8-nllnc I0429 07:01:11.164368 3947 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-7b666bf6d8-nllnc] addLogicalPort annotation time took 131.344744ms I0429 07:01:11.164390 3947 base_network_controller_pods.go:842] Released IPs: 10.129.0.42 for node: ip-10-0-2-36.ec2.internal I0429 07:01:11.164397 3947 pods.go:271] [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc] addLogicalPort took 131.481062ms, libovsdb time 0s E0429 07:01:11.164414 3947 obj_retry.go:716] Failed to update *v1.Pod, old=integration-service/integration-service-controller-manager-7b666bf6d8-nllnc, new=integration-service/integration-service-controller-manager-7b666bf6d8-nllnc, error: addLogicalPort failed for integration-service/integration-service-controller-manager-7b666bf6d8-nllnc: failed to update pod integration-service/integration-service-controller-manager-7b666bf6d8-nllnc: Operation cannot be fulfilled on pods "integration-service-controller-manager-7b666bf6d8-nllnc": the object has been modified; please apply your changes to the latest version and try again I0429 07:01:11.164500 3947 base_network_controller_pods.go:487] [default/integration-service/integration-service-controller-manager-7b666bf6d8-nllnc] creating logical port integration-service_integration-service-controller-manager-7b666bf6d8-nllnc for pod on switch ip-10-0-2-36.ec2.internal I0429 07:01:11.164542 3947 event.go:377] Event(v1.ObjectReference{Kind:"Pod", Namespace:"integration-service", Name:"integration-service-controller-manager-7b666bf6d8-nllnc", UID:"4cbb488f-9a40-4eb4-a8c9-046ae16ec8d5", APIVersion:"v1", ResourceVersion:"26042", FieldPath:""}): type: 'Warning' reason: 'ErrorUpdatingResource' addLogicalPort failed for integration-service/integration-service-controller-manager-7b666bf6d8-nllnc: failed to update pod integration-service/integration-service-controller-manager-7b666bf6d8-nllnc: Operation cannot be fulfilled on pods "integration-service-controller-manager-7b666bf6d8-nllnc": the object has been modified; please apply your changes to the latest version and try again I0429 07:01:11.164617 3947 kube.go:256] Updating pod integration-service/integration-service-controller-manager-7b666bf6d8-nllnc I0429 07:01:11.211267 3947 pod.go:62] [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc] pod update took 46.700452ms I0429 07:01:11.211292 3947 base_network_controller_pods.go:951] [default/integration-service/integration-service-controller-manager-7b666bf6d8-nllnc] addLogicalPort annotation time took 46.731646ms I0429 07:01:11.212445 3947 pods.go:271] [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc] addLogicalPort took 47.953192ms, libovsdb time 696.052µs I0429 07:01:11.262022 3947 namespace.go:142] [build-service] updating namespace I0429 07:01:11.571609 3947 namespace.go:142] [default-tenant] updating namespace I0429 07:01:11.778603 3947 namespace.go:142] [enterprise-contract-service] updating namespace I0429 07:01:11.897907 3947 base_network_controller_pods.go:487] [default/release-service/release-service-controller-manager-6bfdb6bcb6-w6x9m] creating logical port release-service_release-service-controller-manager-6bfdb6bcb6-w6x9m for pod on switch ip-10-0-2-36.ec2.internal I0429 07:01:11.898010 3947 kube.go:256] Updating pod release-service/release-service-controller-manager-6bfdb6bcb6-w6x9m I0429 07:01:11.948669 3947 pod.go:62] [release-service/release-service-controller-manager-6bfdb6bcb6-w6x9m] pod update took 50.681262ms I0429 07:01:11.948700 3947 base_network_controller_pods.go:951] [default/release-service/release-service-controller-manager-6bfdb6bcb6-w6x9m] addLogicalPort annotation time took 50.718063ms I0429 07:01:11.949999 3947 pods.go:271] [release-service/release-service-controller-manager-6bfdb6bcb6-w6x9m] addLogicalPort took 52.117189ms, libovsdb time 891.825µs I0429 07:01:11.979632 3947 namespace.go:142] [image-controller] updating namespace I0429 07:01:12.129940 3947 base_network_controller_pods.go:487] [default/konflux-ui/proxy-8dc549459-x8shb] creating logical port konflux-ui_proxy-8dc549459-x8shb for pod on switch ip-10-0-2-36.ec2.internal I0429 07:01:12.130095 3947 kube.go:256] Updating pod konflux-ui/proxy-8dc549459-x8shb I0429 07:01:12.175191 3947 pod.go:62] [konflux-ui/proxy-8dc549459-x8shb] pod update took 45.131787ms I0429 07:01:12.175214 3947 base_network_controller_pods.go:951] [default/konflux-ui/proxy-8dc549459-x8shb] addLogicalPort annotation time took 45.1608ms I0429 07:01:12.176563 3947 pods.go:271] [konflux-ui/proxy-8dc549459-x8shb] addLogicalPort took 46.638409ms, libovsdb time 899.16µs I0429 07:01:12.201723 3947 namespace.go:142] [integration-service] updating namespace I0429 07:01:12.402419 3947 namespace.go:142] [konflux-cli] updating namespace I0429 07:01:12.570821 3947 namespace.go:142] [konflux-info] updating namespace I0429 07:01:12.760684 3947 namespace.go:142] [namespace-lister] updating namespace I0429 07:01:12.950493 3947 namespace.go:142] [release-service] updating namespace I0429 07:01:13.158385 3947 namespace.go:142] [segment-bridge] updating namespace I0429 07:01:13.281161 3947 namespace.go:142] [build-service] updating namespace I0429 07:01:13.296692 3947 namespace.go:142] [default-tenant] updating namespace I0429 07:01:13.313766 3947 namespace.go:142] [enterprise-contract-service] updating namespace I0429 07:01:13.326624 3947 namespace.go:142] [image-controller] updating namespace I0429 07:01:13.339784 3947 namespace.go:142] [integration-service] updating namespace I0429 07:01:13.363402 3947 namespace.go:142] [konflux-cli] updating namespace I0429 07:01:13.388695 3947 namespace.go:142] [konflux-info] updating namespace I0429 07:01:13.401530 3947 namespace.go:142] [namespace-lister] updating namespace I0429 07:01:13.423970 3947 namespace.go:142] [release-service] updating namespace I0429 07:01:13.435471 3947 namespace.go:142] [segment-bridge] updating namespace I0429 07:01:13.479060 3947 namespace.go:142] [build-service] updating namespace I0429 07:01:13.534513 3947 namespace.go:142] [default-tenant] updating namespace I0429 07:01:13.626189 3947 namespace.go:142] [enterprise-contract-service] updating namespace I0429 07:01:13.659869 3947 cni.go:452] [namespace-lister/namespace-lister-696b855848-5wm97 23d7fa3cb9ffa6300a72016f34e342586d0618759de49c4e93ca2ac395ba10c3 network default NAD default NAD key ] ADD starting CNI request [namespace-lister/namespace-lister-696b855848-5wm97 23d7fa3cb9ffa6300a72016f34e342586d0618759de49c4e93ca2ac395ba10c3 network default NAD default NAD key ] I0429 07:01:13.670438 3947 helper_linux.go:556] ConfigureOVS: namespace: namespace-lister, podName: namespace-lister-696b855848-5wm97, hostIfaceName: 23d7fa3cb9ffa63, network: default, NAD default, SandboxID: "23d7fa3cb9ffa6300a72016f34e342586d0618759de49c4e93ca2ac395ba10c3", PCI device ID: , UID: "8d861f9d-e9f4-42e5-b5b7-4cc676b28f48", MAC: 0a:58:0a:81:00:29, IPs: [10.129.0.41/23] I0429 07:01:13.727610 3947 namespace.go:142] [image-controller] updating namespace I0429 07:01:13.732926 3947 cni.go:473] [namespace-lister/namespace-lister-696b855848-5wm97 23d7fa3cb9ffa6300a72016f34e342586d0618759de49c4e93ca2ac395ba10c3 network default NAD default NAD key default] ADD finished CNI request [namespace-lister/namespace-lister-696b855848-5wm97 23d7fa3cb9ffa6300a72016f34e342586d0618759de49c4e93ca2ac395ba10c3 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"06:47:79:3f:fa:53\",\"name\":\"23d7fa3cb9ffa63\"},{\"mac\":\"0a:58:0a:81:00:29\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/32b9aa92-c679-4a0d-8017-fff7a6985d61\"}],\"ips\":[{\"address\":\"10.129.0.41/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 07:01:13.821733 3947 namespace.go:142] [integration-service] updating namespace I0429 07:01:13.932714 3947 namespace.go:142] [konflux-cli] updating namespace I0429 07:01:14.036795 3947 namespace.go:142] [konflux-info] updating namespace I0429 07:01:14.130007 3947 namespace.go:142] [namespace-lister] updating namespace I0429 07:01:14.222357 3947 namespace.go:142] [release-service] updating namespace I0429 07:01:14.335637 3947 namespace.go:142] [segment-bridge] updating namespace W0429 07:01:18.080966 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service image-controller/image-controller-controller-manager-metrics-service W0429 07:01:18.119851 3947 util.go:870] Failed to build global endpoints for port TCP/https: empty IP address endpoints for service build-service/build-service-controller-manager-metrics-service W0429 07:01:18.903782 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service namespace-lister/namespace-lister I0429 07:01:19.188311 3947 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-kv285-fhbqg of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:01:19.189006 3947 obj_retry.go:498] Detected object tsf-tas/tuf-repository-init-kv285-fhbqg of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:01:19.198830 3947 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc ae083d76633cd9fe84c9c9253eaf41d4875d2b7434b140e0bd189a5876ff8595 network default NAD default NAD key ] ADD starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc ae083d76633cd9fe84c9c9253eaf41d4875d2b7434b140e0bd189a5876ff8595 network default NAD default NAD key ] I0429 07:01:19.208475 3947 helper_linux.go:556] ConfigureOVS: namespace: integration-service, podName: integration-service-controller-manager-7b666bf6d8-nllnc, hostIfaceName: ae083d76633cd9f, network: default, NAD default, SandboxID: "ae083d76633cd9fe84c9c9253eaf41d4875d2b7434b140e0bd189a5876ff8595", PCI device ID: , UID: "4cbb488f-9a40-4eb4-a8c9-046ae16ec8d5", MAC: 0a:58:0a:81:00:2b, IPs: [10.129.0.43/23] I0429 07:01:19.263414 3947 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc ae083d76633cd9fe84c9c9253eaf41d4875d2b7434b140e0bd189a5876ff8595 network default NAD default NAD key default] ADD finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc ae083d76633cd9fe84c9c9253eaf41d4875d2b7434b140e0bd189a5876ff8595 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"82:9f:ad:60:c5:35\",\"name\":\"ae083d76633cd9f\"},{\"mac\":\"0a:58:0a:81:00:2b\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/a1355e69-935b-4340-94d1-062dbf18bdae\"}],\"ips\":[{\"address\":\"10.129.0.43/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err W0429 07:01:19.920237 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service namespace-lister/namespace-lister I0429 07:01:19.995840 3947 cni.go:452] [konflux-ui/proxy-8dc549459-x8shb 47124550bb77f9d895d99a1d5f4469de0ec7e090a287c599f834d803d928d11c network default NAD default NAD key ] ADD starting CNI request [konflux-ui/proxy-8dc549459-x8shb 47124550bb77f9d895d99a1d5f4469de0ec7e090a287c599f834d803d928d11c network default NAD default NAD key ] I0429 07:01:20.006187 3947 helper_linux.go:556] ConfigureOVS: namespace: konflux-ui, podName: proxy-8dc549459-x8shb, hostIfaceName: 47124550bb77f9d, network: default, NAD default, SandboxID: "47124550bb77f9d895d99a1d5f4469de0ec7e090a287c599f834d803d928d11c", PCI device ID: , UID: "1a8abec3-a548-40ae-ad7e-7bd2025bba8d", MAC: 0a:58:0a:81:00:2d, IPs: [10.129.0.45/23] I0429 07:01:20.059654 3947 cni.go:452] [release-service/release-service-controller-manager-6bfdb6bcb6-w6x9m b7ceb8931fce2acec8a3bfa0f4a8d5b85b8fe2052c89cb739609a867ed19f334 network default NAD default NAD key ] ADD starting CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-w6x9m b7ceb8931fce2acec8a3bfa0f4a8d5b85b8fe2052c89cb739609a867ed19f334 network default NAD default NAD key ] I0429 07:01:20.064292 3947 cni.go:473] [konflux-ui/proxy-8dc549459-x8shb 47124550bb77f9d895d99a1d5f4469de0ec7e090a287c599f834d803d928d11c network default NAD default NAD key default] ADD finished CNI request [konflux-ui/proxy-8dc549459-x8shb 47124550bb77f9d895d99a1d5f4469de0ec7e090a287c599f834d803d928d11c network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"0a:e7:81:3d:c3:c0\",\"name\":\"47124550bb77f9d\"},{\"mac\":\"0a:58:0a:81:00:2d\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/59d7407c-1574-401b-b13a-87765d009589\"}],\"ips\":[{\"address\":\"10.129.0.45/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 07:01:20.069341 3947 helper_linux.go:556] ConfigureOVS: namespace: release-service, podName: release-service-controller-manager-6bfdb6bcb6-w6x9m, hostIfaceName: b7ceb8931fce2ac, network: default, NAD default, SandboxID: "b7ceb8931fce2acec8a3bfa0f4a8d5b85b8fe2052c89cb739609a867ed19f334", PCI device ID: , UID: "2ad466be-e820-4589-81fb-785053e2ae7d", MAC: 0a:58:0a:81:00:2c, IPs: [10.129.0.44/23] I0429 07:01:20.127903 3947 cni.go:473] [release-service/release-service-controller-manager-6bfdb6bcb6-w6x9m b7ceb8931fce2acec8a3bfa0f4a8d5b85b8fe2052c89cb739609a867ed19f334 network default NAD default NAD key default] ADD finished CNI request [release-service/release-service-controller-manager-6bfdb6bcb6-w6x9m b7ceb8931fce2acec8a3bfa0f4a8d5b85b8fe2052c89cb739609a867ed19f334 network default NAD default NAD key default], result "{\"interfaces\":[{\"mac\":\"8e:7a:eb:91:af:8b\",\"name\":\"b7ceb8931fce2ac\"},{\"mac\":\"0a:58:0a:81:00:2c\",\"name\":\"eth0\",\"sandbox\":\"/var/run/netns/02995ba3-1ada-4ca7-aac9-894c88be0aa6\"}],\"ips\":[{\"address\":\"10.129.0.44/23\",\"gateway\":\"10.129.0.1\",\"interface\":1}]}", err I0429 07:01:20.215093 3947 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-kv285-fhbqg of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:01:20.215118 3947 obj_retry.go:492] Detected object tsf-tas/tuf-repository-init-kv285-fhbqg of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed W0429 07:01:21.078790 3947 util.go:870] Failed to build global endpoints for port TCP/dex: empty IP address endpoints for service konflux-ui/dex W0429 07:01:24.119156 3947 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0429 07:01:24.119177 3947 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0429 07:01:24.151829 3947 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0429 07:01:24.152029 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0429 07:01:25.967487 3947 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service release-service/release-service-webhook-service W0429 07:01:25.968378 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service W0429 07:01:25.999481 3947 util.go:870] Failed to build global endpoints for port TCP/: empty IP address endpoints for service integration-service/integration-service-webhook-service W0429 07:01:26.000368 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0429 07:01:26.033676 3947 util.go:870] Failed to build global endpoints for port TCP/web: empty IP address endpoints for service konflux-ui/proxy W0429 07:01:26.033701 3947 util.go:870] Failed to build global endpoints for port TCP/web-tls: empty IP address endpoints for service konflux-ui/proxy W0429 07:01:31.160095 3947 util.go:870] Failed to build global endpoints for port TCP/http: empty IP address endpoints for service tsf-tas/tuf W0429 07:01:35.148867 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0429 07:01:35.148891 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service integration-service/integration-service-controller-manager-metrics-service W0429 07:01:35.149283 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service W0429 07:01:35.149306 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9443: empty IP address endpoints for service integration-service/integration-service-webhook-service I0429 07:01:35.339827 3947 cni.go:452] [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc ae083d76633cd9fe84c9c9253eaf41d4875d2b7434b140e0bd189a5876ff8595 network default NAD default NAD key ] DEL starting CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc ae083d76633cd9fe84c9c9253eaf41d4875d2b7434b140e0bd189a5876ff8595 network default NAD default NAD key ] I0429 07:01:35.415461 3947 cni.go:473] [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc ae083d76633cd9fe84c9c9253eaf41d4875d2b7434b140e0bd189a5876ff8595 network default NAD default NAD key default] DEL finished CNI request [integration-service/integration-service-controller-manager-7b666bf6d8-nllnc ae083d76633cd9fe84c9c9253eaf41d4875d2b7434b140e0bd189a5876ff8595 network default NAD default NAD key default], result "{}", err I0429 07:01:35.473751 3947 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-nllnc of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:01:35.473789 3947 pods.go:176] Deleting pod: integration-service/integration-service-controller-manager-7b666bf6d8-nllnc I0429 07:01:35.475217 3947 pods.go:236] Attempting to release IPs for pod: integration-service/integration-service-controller-manager-7b666bf6d8-nllnc, ips: 10.129.0.43 I0429 07:01:35.475250 3947 obj_retry.go:498] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-nllnc of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:01:36.031947 3947 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-nllnc of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:01:36.031975 3947 obj_retry.go:492] Detected object integration-service/integration-service-controller-manager-7b666bf6d8-nllnc of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:01:36.039995 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod integration-service/integration-service-controller-manager-7b666bf6d8-nllnc I0429 07:01:36.040023 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod integration-service/integration-service-controller-manager-7b666bf6d8-nllnc W0429 07:01:36.978968 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9443: empty IP address endpoints for service release-service/release-service-webhook-service W0429 07:01:36.979390 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service release-service/release-service-controller-manager-metrics-service I0429 07:01:53.896771 3947 namespace.go:142] [openshift-pipelines] updating namespace W0429 07:02:01.173918 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0429 07:02:01.173937 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0429 07:02:01.173947 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0429 07:02:01.173952 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy W0429 07:02:06.050265 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 8080: empty IP address endpoints for service konflux-ui/proxy W0429 07:02:06.050300 3947 util.go:959] Failed to build node endpoints for node ip-10-0-2-36.ec2.internal port 9443: empty IP address endpoints for service konflux-ui/proxy I0429 07:02:23.195618 3947 reflector.go:979] "Watch close" reflector="github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/informers/externalversions/factory.go:117" type="*v1.NetworkAttachmentDefinition" totalItems=6 I0429 07:02:36.354368 3947 obj_retry.go:498] Detected object konflux-ui/proxy-688676554b-bsqwd of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:02:36.354957 3947 obj_retry.go:498] Detected object konflux-ui/proxy-688676554b-bsqwd of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:02:36.518964 3947 obj_retry.go:492] Detected object konflux-ui/proxy-688676554b-bsqwd of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:02:36.518981 3947 obj_retry.go:492] Detected object konflux-ui/proxy-688676554b-bsqwd of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:02:36.530862 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/proxy-688676554b-bsqwd I0429 07:02:36.530887 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/proxy-688676554b-bsqwd I0429 07:02:40.378698 3947 cni.go:452] [konflux-ui/test-tsf-konflux d84e18157ce3b97f4fd828acae9c54c6d6bd49cc39b363ff8460aaa0d7c5af10 network default NAD default NAD key ] DEL starting CNI request [konflux-ui/test-tsf-konflux d84e18157ce3b97f4fd828acae9c54c6d6bd49cc39b363ff8460aaa0d7c5af10 network default NAD default NAD key ] I0429 07:02:40.479288 3947 cni.go:473] [konflux-ui/test-tsf-konflux d84e18157ce3b97f4fd828acae9c54c6d6bd49cc39b363ff8460aaa0d7c5af10 network default NAD default NAD key default] DEL finished CNI request [konflux-ui/test-tsf-konflux d84e18157ce3b97f4fd828acae9c54c6d6bd49cc39b363ff8460aaa0d7c5af10 network default NAD default NAD key default], result "{}", err I0429 07:02:40.537961 3947 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) during update event: will remove it I0429 07:02:40.538000 3947 pods.go:176] Deleting pod: konflux-ui/test-tsf-konflux I0429 07:02:40.539604 3947 pods.go:236] Attempting to release IPs for pod: konflux-ui/test-tsf-konflux, ips: 10.129.0.40 I0429 07:02:40.539633 3947 obj_retry.go:498] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) during update event: will remove it I0429 07:02:40.558665 3947 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *v1.Pod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:02:40.558685 3947 obj_retry.go:492] Detected object konflux-ui/test-tsf-konflux of type *factory.egressIPPod in terminal state (e.g. completed) will be ignored as it has already been processed I0429 07:02:40.567738 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *factory.egressIPPod konflux-ui/test-tsf-konflux I0429 07:02:40.567763 3947 obj_retry.go:771] Ignoring delete event for resource in terminal state *v1.Pod konflux-ui/test-tsf-konflux I0429 07:02:48.159804 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressIP" totalItems=7 I0429 07:03:08.167571 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressQoS" totalItems=8 I0429 07:03:09.174375 3947 reflector.go:979] "Watch close" reflector="github.com/openshift/ovn-kubernetes/go-controller/pkg/crd/egressservice/v1/apis/informers/externalversions/factory.go:140" type="*v1.EgressService" totalItems=7 I0429 07:03:13.066648 3947 reflector.go:979] "Watch close" reflector="k8s.io/client-go/informers/factory.go:160" type="*v1.Service" totalItems=84 I0429 07:03:23.150965 3947 reflector.go:979] "Watch close" reflector="sigs.k8s.io/network-policy-api/pkg/client/informers/externalversions/factory.go:141" type="*v1alpha1.AdminNetworkPolicy" totalItems=9